From e3b6254090e5e6a3bec017ce88a7f397ba6ec48d Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Thu, 22 Jan 2026 23:28:56 +0800 Subject: [PATCH 01/83] debug Qwen TTS (#902) Signed-off-by: Taichang Zhou --- .../models/qwen3_tts/modeling_qwen3_tts.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vllm_omni/model_executor/models/qwen3_tts/modeling_qwen3_tts.py b/vllm_omni/model_executor/models/qwen3_tts/modeling_qwen3_tts.py index ac9844f89..75fe5dbf4 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/modeling_qwen3_tts.py +++ b/vllm_omni/model_executor/models/qwen3_tts/modeling_qwen3_tts.py @@ -43,6 +43,8 @@ from transformers.processing_utils import Unpack from transformers.utils import can_return_tuple, logging from transformers.utils.hub import cached_file +from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific + from .configuration_qwen3_tts import ( Qwen3TTSConfig, Qwen3TTSSpeakerEncoderConfig, @@ -1844,6 +1846,15 @@ class Qwen3TTSForConditionalGeneration(Qwen3TTSPreTrainedModel, GenerationMixin) weights_only=weights_only, **kwargs, ) + if not local_files_only and not os.path.isdir(pretrained_model_name_or_path): + download_cache_dir = kwargs.get("cache_dir", cache_dir) + download_revision = kwargs.get("revision", revision) + download_weights_from_hf_specific( + pretrained_model_name_or_path, + cache_dir=download_cache_dir, + allow_patterns=["speech_tokenizer/*"], + revision=download_revision, + ) speech_tokenizer_path = cached_file( pretrained_model_name_or_path, "speech_tokenizer/config.json", From 75dddcb2b5c1d7f88fc056f6ba91627d5ac518d7 Mon Sep 17 00:00:00 2001 From: TJian Date: Fri, 23 Jan 2026 05:55:50 +0800 Subject: [PATCH 02/83] [ROCm] [Doc] Add instructions to install ROCm dependencies to run Qwen3 TTS model (#905) Signed-off-by: tjtanaa --- docs/getting_started/installation/gpu/rocm.inc.md | 9 --------- examples/offline_inference/qwen3_tts/README.md | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/getting_started/installation/gpu/rocm.inc.md b/docs/getting_started/installation/gpu/rocm.inc.md index a98a3cbe9..1a8ffb612 100644 --- a/docs/getting_started/installation/gpu/rocm.inc.md +++ b/docs/getting_started/installation/gpu/rocm.inc.md @@ -23,15 +23,6 @@ vLLM-Omni current recommends the steps in under setup through Docker Images. DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.rocm -t vllm-omni-rocm . ``` -If you want to specify which GPU Arch to build for to cutdown build time: - -```bash -DOCKER_BUILDKIT=1 docker build \ - -f docker/Dockerfile.rocm \ - --build-arg PYTORCH_ROCM_ARCH="gfx942;gfx950" \ - -t vllm-omni-rocm . -``` - #### Launch the docker image ##### Launch with OpenAI API Server diff --git a/examples/offline_inference/qwen3_tts/README.md b/examples/offline_inference/qwen3_tts/README.md index 266227c4b..eab66fe14 100644 --- a/examples/offline_inference/qwen3_tts/README.md +++ b/examples/offline_inference/qwen3_tts/README.md @@ -13,6 +13,15 @@ Qwen3 TTS provides multiple task variants for speech generation: ## Setup Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +### ROCm Dependencies + +You will need to install these two dependencies `onnxruntime-rocm` and `sox`. + +``` +pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm +pip install onnxruntime-rocm sox +``` + ## Quick Start Run a single sample for a task: From b8499e4839dea847bc363f808b16c40d46ae6472 Mon Sep 17 00:00:00 2001 From: Roger Wang Date: Thu, 22 Jan 2026 17:09:48 -0800 Subject: [PATCH 03/83] Bump version to 0.14.0rc1 (#910) Signed-off-by: Roger Wang --- docs/getting_started/installation/gpu/cuda.inc.md | 2 +- pyproject.toml | 2 +- vllm_omni/version.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/getting_started/installation/gpu/cuda.inc.md b/docs/getting_started/installation/gpu/cuda.inc.md index 5331d2d34..dbc528327 100644 --- a/docs/getting_started/installation/gpu/cuda.inc.md +++ b/docs/getting_started/installation/gpu/cuda.inc.md @@ -17,7 +17,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new** # --8<-- [start:pre-built-wheels] #### Installation of vLLM -Note: Pre-built wheels are currently only available for vLLM-Omni 0.11.0rc1 and 0.12.0rc1. For the latest version, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source). +Note: Pre-built wheels are currently only available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1. For the latest version, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source). vLLM-Omni is built based on vLLM. Please install it with command below. diff --git a/pyproject.toml b/pyproject.toml index 6c4b2c23a..279d8e7d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "vllm-omni" -version = "0.12.0rc1" +version = "0.14.0rc1" description = "A framework for efficient model inference with omni-modality models" readme = "README.md" requires-python = ">=3.10,<3.14" diff --git a/vllm_omni/version.py b/vllm_omni/version.py index 848aa7711..0a231e9bd 100644 --- a/vllm_omni/version.py +++ b/vllm_omni/version.py @@ -1,3 +1,3 @@ -__version__ = "0.12.0rc1" -__version_tuple__ = (0, 12, "0rc1") +__version__ = "0.14.0rc1" +__version_tuple__ = (0, 14, "0rc1") # TODO: add auto version generation From 28d13e0dee41df8dc64e624c6c03357961e4661d Mon Sep 17 00:00:00 2001 From: TJian Date: Fri, 23 Jan 2026 11:37:54 +0800 Subject: [PATCH 04/83] [Doc] Fix the version in the documentation to v0.14.0rc1; fix dockerfile.rocm entrypoint (#913) Signed-off-by: tjtanaa --- docker/Dockerfile.rocm | 6 +++++- docs/getting_started/installation/gpu/cuda.inc.md | 2 +- docs/getting_started/installation/gpu/rocm.inc.md | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 5160c45a2..0f5923342 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -17,6 +17,10 @@ RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni COPY . ${COMMON_WORKDIR}/vllm-omni RUN cd ${COMMON_WORKDIR}/vllm-omni && uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]" +# When we are installing onnxruntime-rocm, we need to uninstall the system-installed onnxruntime first. +# These are the dependencies of Qwen3-TTS. +RUN uv pip uninstall onnxruntime --system && uv pip install --no-cache-dir onnxruntime-rocm sox --system + RUN ln -sf /usr/bin/python3 /usr/bin/python CMD ["/bin/bash"] @@ -25,4 +29,4 @@ ENTRYPOINT [] #Set entrypoint for vllm-openai official images FROM final AS vllm-openai -ENTRYPOINT ["vllm", "serve"] +ENTRYPOINT ["vllm", "serve", "--omni"] diff --git a/docs/getting_started/installation/gpu/cuda.inc.md b/docs/getting_started/installation/gpu/cuda.inc.md index dbc528327..c4b34ab15 100644 --- a/docs/getting_started/installation/gpu/cuda.inc.md +++ b/docs/getting_started/installation/gpu/cuda.inc.md @@ -93,7 +93,7 @@ docker run --runtime nvidia --gpus 2 \ --env "HF_TOKEN=$HF_TOKEN" \ -p 8091:8091 \ --ipc=host \ - vllm/vllm-omni:v0.14.0 \ + vllm/vllm-omni:v0.14.0rc1 \ --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091 ``` diff --git a/docs/getting_started/installation/gpu/rocm.inc.md b/docs/getting_started/installation/gpu/rocm.inc.md index 1a8ffb612..60ad1f2b2 100644 --- a/docs/getting_started/installation/gpu/rocm.inc.md +++ b/docs/getting_started/installation/gpu/rocm.inc.md @@ -80,7 +80,7 @@ docker run --rm \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=$HF_TOKEN" \ -p 8091:8091 \ - vllm/vllm-omni-rocm:v0.14.0 \ + vllm/vllm-omni-rocm:v0.14.0rc1 \ --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` @@ -99,7 +99,7 @@ docker run --rm -it \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=$HF_TOKEN" \ --entrypoint bash \ - vllm/vllm-omni-rocm:v0.14.0 + vllm/vllm-omni-rocm:v0.14.0rc1 ``` # --8<-- [end:pre-built-images] From 8f4f091387ba734f3fd3e63e409bea8b155580f7 Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Fri, 23 Jan 2026 12:14:48 +0800 Subject: [PATCH 05/83] [Misc] update wechat image (#914) Signed-off-by: David Chen <530634352@qq.com> --- docs/assets/WeChat.jpg | Bin 231347 -> 129258 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/assets/WeChat.jpg b/docs/assets/WeChat.jpg index 5a63afde85c697f3c8d8b2f25d0d3e647e28ed4b..5a15b524cd21a20d4d267812c9534f66b6c8ae15 100644 GIT binary patch literal 129258 zcmd422UJsA*EYICklv-YprD|DC`F`(B2A7IQ9_S`h=_=&^b(4IbU^_@r6~d;M4D8I zbP$nFXi@@7hlCnJ%H6)_E$2JmH}3fFfA6^C4`TsIc2@RWYnEp|b1vE#Z5H6VWN2at zKp+4B0e=A6B%lv4&>vwqLeIc(gn^Nffr*8Sg@u`!g_nbqjZ1)6P*8xEpI=z)1WZ^& zQk0)xLQX>Rr1YuNr-WegXXRzio{%{u^Lr8qBO@aV6U$K+mZLJl{K7K-j~}!z05=nq z4SEa;IS$ZqL!jIcS{ncZ@AC-c9}nQ44+tF;ykABpW)@cPf@&^+4g!VJ(L;|Mp{ED0 z4guc>=(&&Z96P7WaMbb!<8g0Z#qg9oCP}^WX1*%}cqt|42NBFH`~reP!Y58j%bYr` zta4uUf||PiMFT@4W0Ol(SFc$kY;5gZZo1xbyX}7G;UgbkKmUNh$j4F9F;8OSQlC9f zOV4+BVVu8Q^6zp$uaf5Qv(bGfe8Gp+Kq4NU=l$-v@v2zSOx|WPL zypJAN3}@oiOUWy5W|maCg6DI7Fu=kur95|n@LRNhNcO)bSj7L9Wd9cIKXOd~Y)}Ze zc~EWu4v^l($v!7$%)eHcqXE&bqbd>KR0Hno~#2on!4(831oYV;PIS6N7;x;+PSMB7c zJ=4DaG!ZvS zFNl?|Iz-j1Bj#-HIVU*^*hZhcUwVrBX;`*Bq{15;2av?$(1p|XE9d#KqlzC!GtSBN zfBh^bNcTbOiNr`8AXB8UDZg_l@^W%KmDT?CgnPlmsEZb+=k|c3PEfzYt3+#&%r`NH zqtdaW?=R8ckpl2v_`CWX#BW2F>}0!}_{k6Pi*540m&FICw_0d`TPB;IE-Dx}5s@bS zl;M6*i0tTXAU{q99C_%KJv)ZPfcGnvOmd!*il)bezQC~605ZSH$7`adW$pz#tE~hN zL>!?_0|Fapz+)O9NQM(mZfx}Z5x4T4`a^T3aw@!TewP0Vvmkv~XnvU1nK%NcFm+Iz z58wG@Q`x6tSVc%)v|VnrnT{FKs_;!$iaYWeS>f;xqPacRQ+a$%MPAdPHu0V`lcmkh z_d#=MC?S{A?;p2U9_)(q)MybGYZ$H0{KMC`Ctc!Zx#kgQoitseDpk0l6}AJ6VlH4n z{Uj;IZEu1%$5#c9_1$+fF^#t`xuQAg5vCfYC3yX*k=>3kAd}zNibHoKbCT7EJ$F`# zXVB=M=I3%hvA%e^BWDxdr)j}hIDphJIJEY6EHHScKWb?TI2_!R**zRt6+cZ(1Z|I< z$fLmT=#W{}^VrJcCtG#v*%OvV#Xxo-5F!ID&u@ANnr9illLGkUd{>~J{%kc@*!F|9 zlPcCQQq_?@ez#lTz5^f*>878Th=8bx#|(yP#K|u98toP03rZ<%{3NpyAGUjzZaU#N z0m>|^8{>4nc{EC#tTNxneJ0k^BdBotYeBAnz4tjFHMdxE_Gm87s6x*@x5-C%z=a)d8POu5&E2hN_$n{JlU3yFq|k3p%k$ zj2K_iS1_g<;1V4BOA4Wj3VgHadD?hBse0uRU}?bnpW(!Z!%QGsvN$jS?+jduR$#Xt zikRQyI(B22y-l3Mz+`r2!nCDHiolosbk8pO)&Aw5om$^}U-q#Sx=Y<8=!nl71fbi| z3F?fkwriz67s|X8HZpqoN59-N-qP`v(`l9npNU*{3If*|IhRZVJdm%DYs5gpm6|&>X*Ajf1 zn3qR38Qj>Y3Rh{|OE}VA`UNu2s9~QF!NF;+@8nk+IvmO~v}^j()Kk@}DOy0L=Ik2t zNsTfa(NJE<<_;o(tVPKSJ~t2NnOsVlK)Pd9;d1N`ct0eME54o03G62p5?Wg^oGLG> zUQPz}9ByUbET4#?bX}0GEf-Dcw^#;Ya*ye^jM<+h z-`O=64L0i6JT^Ot;+g)Ao2 zUwZJ@xR5-bOcHJUOI$bw#s$or3vr$rqwt@&@ZY8b!_5EL(>*gJ*37RTr<_?Q+kW_C zwWFsYy)Gg7(bEcQ<_;PiN=J77L`=N3ryghEfs?(teIi=(ahf8-PTw${#;q`|z&M#) zx_^x(HOeO+(_+nsdG2!aYZmS^cP>Jyb|$hVJV6AW`Jo)UL*XV_N;PVs*d4XU5KjY) z`gCCM0fT$~3a!G)37ST3`ODlH^|g0zMp&zC^d^O|0Im5=4HcB<%8?$NlbS|FA%)ks z|FD$M8(X{=&=GP6MZCEVrABL6Q7J}LUI<~GZ%=@%=o?N0^u>#5KesY z+%-Skg%K?nJF5_5NPPDMBZV!nkU~l(%YdHCtRKGQG6LTmM$PHK#0QW!P_5FpVLRtm za`so?_%~D`4INNRlCRPLj3tN>fZ!m55YXt@Ugx6vJKv*bK;Y1l{NL_nx2*b5dhATb z{YPG0KR)FOoGtz4@$*@>Zl{(K5!)j?ofeRPXFu>(A+$Nj<5--{*!2i_;r*aE;0@BA z+DdvoJYf{K(+xYHUTNRQGoR*8AAYby86@oPqEls8a>iUX{jtlgpph*>SL$W*!?c+w zCGdq>kDT4FlQiJsi2%Nk9zyFJBAjemHtC@)N#NBM!@Op@yX9aYSU<)Kjb-|&3r)7n z+Fn<$?aM(cY=woO2z;Hyb%}W$%4V9=H4OYX+1RJ5e1QVS!p1kF6Xr=QO zTKU4IG$x8DT@{z=-!Zu&?_oEjeBkJp@A$)r+jaS=U_u-)lK)Ye1_%@FEgR#8%n4^= z{d`VMik?NbOV;hMh@7lh0??Y2Jk&yhe+l(-3^l=SNBuiNJkbRw#<6ca;=KK#Z2sJ( z%nqaST}Fm5xI$3Ye$H4Y_0s1}N(8tk10S&_3SU4?$+w2#D@%_)+j_a`kbJLtWri+P z*k7YX3AvKZKZ?U{6u{hqsb1JFg7opVUrU^C)5{IC5Eftrb&_*p#V=iK4NPY)bhGH*#`K)&m^M@4~XQ!Id=j_fgQhSlN zmZ+!j)S5@56XzQW?g+3%8^#T8sJ)Pp;6G*}aWg!}+X2>|6A{SS7GgGYrJ=rlXqRcI zC!q9;mkOhf?tId1=r*@x*{u--G0@$(#xI+e9A1kOW)79$R470cmmZGMUbU_9-;vT zmmQ7BiYQQ5mz+mXzuk#bA--qMH)fWT!^z#6!4&i9*xn)mQpaG7*wMP+wxdX32N0)gmW?}7y*RQAaNU=O=Ig5IHw41KpD;fp_!)FzE z-qrE#{(2L{lNQ;>Qo=0}@kPNZ!bp1uvi8XORwYhMK>v1z-N?@Rg|EyBj#Fk-S%}ltbKX6<@SlHPa zC2l(yOJ3~$(!&ERgcCM-&ofwoJ%K8gRWNhXP=K9iWG0 zCqp#avgk**lvV4U@*sKxwKvmM#a{T(i|02veI^Rvn`DaznxyRQ(^4|JgC;!(W+_9@ zUM0QiXXHFQj_STy1!}t*4cHMreBWq4>jt9vBmT&1jU!|gbURLDSeNs)Sla_upSugr ziaPR1)K`e2Rc)DmWz)2p(du3W^V*v{atJ@~wssSVvuwH@x-j^ZG` z>`a`N@YaUI!F~JbPRS;SG>hP~FVgtZ6%Z?fX-L^kvJU>vD;Ph~Kg34{MSMRtP@Sk` zU+wW)z_Mc4bL5++{M%JU27u>nvE&7b`6&y=U&#sZ1!Hz_+yhA5vR< zXGkg@t7v}ew`pN4SwHB#Zt=`AZbe_#zU_V2hXARgebTS2$YO`eM6cFt$;y#;!<*Gd zZ{uucGp||s|Fl_OjQlPG?ZS8>4^9UnDB;eac5Gw&c1fHBzu8|ggfqCWTT~pb)1um3 z%E3O&v^`ai%YU|9;xKC){g-&8BiaYv-nU>C8h|`S1Kt#%Gdb-juo0>-(eDb;H=E7F zE#5f#>_<;$r}IL`wx<@J^*Lm1DG+bc08VhLJKxiQf5f;jwZ5@uo5V?Ktvzb}&Os=~ zAoM8t%s{A!dxr0X!?UR=r?aX8O0!DguS>$eK&7|Q!UXsiO!O%n>(6JM=O>g7GS71c z++!0GnvBs1TTB4=WDZFO#+Hx%rqmX08lb+6@qiu32cjuKU<_L-K;I`C;Df#htH$F- z9(nP$;$&Bq72D|?TBuexA;-8={lBZtpR7m4%|Ml5PXZSLKllEa&O!Zr@-yhZ;2(@E z#`k8u+ks$Cq&c-8w&j__2479gpr&LkuVY*lIE*weYkz)kD*W-xOKv~5Eu~476&O!2 zbS{UB48MHTHbwb|v{@qi=gurHJ9@|WeyW13ouIW2A+J;0;9IAN8B~l5xEnsbC@}Qf zgSM~NLj$0tpt9_PsJB1#NZ-yrxM{21|3fM5;J&)Ij#zI1W?cY8 zrm!RA%RV;`Pu_N^`H>VaS1jT(7SpV)({u|JM-%eq(LsJexfCHWXD!T?J z-rW4Xm+imzvi48B@?T#F)7H#yYW@3%|Ni{{hshc_=Z#uE8_i{x^FF-1*L!BE{8K%X zp*NX*>?|xt8M4SXMdlpds6xkQAIl|jrJy8tygJ4}=A zbLE~cI7zDj#lMdRRA6JVpF+FfTQ)a9$0S4p_UwQ|Fjoh`@2n<*5c8#pZ^ZucX!TP5 zXLAg%rOCMj()o4`^YUNin7O~1S zyrBw<%;iwnz$g$V03+WE({hg6LJ@C!MWc4kVyJ)vick;#pC&7lZAS4nip-m0EAbmO ze7zHi)21Ex2F_~;Ag86l4v^jM*?R(tGJ?a94v-Z7YLW;t8(C#UPDk_60PI)L(yDVg zFQDI}ZX|!6M;2sIvFNmt{|REUQ9j&rdpYk^Dmr8$bzlEpitM@LD(7Um-h7<;c3&ek z2%C%;K~*7};r#wNIJPDUe0!sM;^!S>src{4P7DTDo2S2=m7rtVr)Q%J?wH*3A*QLc zS$5~^(rHAw-)su|a6PBdx9)x??%buu-mxay^eYr3J_MlVb1 zH*={}6Dh2zt@hDD_(x5JH%nB%33%mNUgh?uyW5&o%nkRV%Mmm z#P`1j-6d*cE;Xc{a(zfwUxgHc`z(EkY9VzbFH&+2^}c(UYuus% z3g7e^J}tNjeOHv7n?#ReaaUCp+r?w)32)MD-!z3ug^AvSygJY6q@$f{0mZnoX66yc z-*mDKc8cPsq>4W1q+C4nj;v?~g`VQvagp79a;0lwUQwQ#!V<++qdEIjN z&^IZsLz#({tuU@U0%PoKj9Jac*fwwG)LXUg+|QscT)EIx^N+wnN+zLU7Libmep3Ig za<0e5M?>C#Opcw8%DW3##N5cv?Al$|ft{c1Q>h~KdC8?*7GX5uj;r{nM$g?JIgPQb z`s6kwFEm!EEMp=q^*X;vXb|P5@We0hb@gXzCPR*gWVrlr#{I72PV3bUaiD2D3BO?&M8Vo~Woca#W1;Yrc5sl_W9!)ZFvSOp9 zZhRk8PWVlAwRamzKq>`S9t65P<;>ESb-3(9P;GzU&?{BJe&wgCaeLB8N}8ehbpYBA zvCZbL%^qsE)7+~xpe=`HLED9nvQtXA=lJp%4cv^b@NKr+c{sS zd^e{J*B*Vp&>KASe(JJcvE!6uuhhQMA6q7JA8cR8N~M}hJ^Svlh8Yiaj$E*|^vw$(So^5O~ot`lO- zR^OS{bekpV!@OY(XKqy!E9aQy-=zw8UYS-LD8F*;ElW?zSg}OXYzp!%A0em3JWTt1 zIWZ>xj*~=XjOozM>WfV$>)IrjV~MRyBbKo}+{3ewXWbC<@#p^`*bI0fcx_8)cB&LHIRMAW zHmP4Uv2sd8Jz$`vU(v9*YV$@dS8jxsvBdSGi|O#n!nXMMv17FZv(?c?EZf~dta8`0 zZK58uanu4?z>fLE`@IchR1PP7hE;7{tm_Nq#|LG{5~95}VVQN>g`-XOi=$1m!~=XG zg5U;*NDga1gZF9zxua5R-KVS&;1ATh{F_Q-9t)ON@Ff>3k zm(!BSS2W#vsNlbGv9Z>u%A@&i&oKt>y+?PCQDK=c2vg67v0vL3uv0TEO-EO<1e)2s zTI$<}&y&(WN&AgywpS}&ERT{BRTI&@F#n2eUV704WVgLyEqjljjA8o^Vr(8=cd#`$fDxp2jT?`dPnS}8yDA7w^3Y?L$s)7tCS z3KkV6rko739V&`tgv1;eIwg2=d6W_s83jS-^8q^mFPCrYnWy;vaD8gu{Z2?+qSHgM zN6=a_PeKU@x~KiFc9ZCq6ASKMN&pfjd2xd~KM2^j(!s3CX;QtW0hbk+)u%F(0}<>?!%+c)e+n3lI9%#CMh) zi$L+kQs5yRa?kA(v3btQQ#nOx`t4oIxDBgjtEAkde2jtY{7l3QgzWZ~d~vYai`!2B zoz3!Kd}@dPFL&F(r2IZLL z;?(+r&XfHLcjt_mC~Q-JHoNC(8d+4-30O8CW05|%l*VSpJ6EC9DL7vW1T%Q7nU7`t z3J4a%F-fHxMvDxedvoq|rrCWBH$Gerz7KvBzT{+1<^*Yl`LrGE+7N!#niw>vu|(Dx zI6=+3kXicXI@Q>+itfwoGi(xLaW3a|^eIX)Y9WgFY}Jzn6r4kp{Lf$*ZuyAw0OCHg zyAGp9oTgax!xxSeqOesc5>v?zU}>X$ju09-`$NsE(X?3K7uUzfy0Hso8-&`lq` zJse{D6wvBMTSV9_#-4~8nZGY&Yo;q+1&|IJ_I`lM25-&TrP2VB9ClD;+i$&r1{_Dl zz*i|?3T&IpnJimEybO{;=FKyELKR@zvFuRbgJQ-k6caV5t>=*>Eg9&}yMOVp;9mgS zj{H z(_VV#aL!oj#_%a+Qrs$4tAGaFrvaw{a!q4h*49Q)NwKhf>|1v!-<5A>Ch#kFPG7PE zJpWmh5ks^z6t7WF5bZo{g-^k&)`uSKP7EHoL5wg0G3K>{*Z4{5C50%=nyd$i!P7B+ z+y>Bn3KNo4Q$!q?M*l?Z*kPAELjytwU|V*L2rzS%r2+Y)VVbz6VoJ33eqv}k^3bWt z`meAxm-GG7@BAGc1k6NCr*LOPFt?j@%rE|ZKP=o$Rj|h}&(t-@EdlWriLav`2Q>g& zgqqKpLmGun2>bk83{x(Nw+}cGJ$U%-b^lGn0w(&yn{hxU=(J1GJuV5`aeHzodFX>a@g&22=kwX zs^3fV0K-exe>i6zQ~#^xddlaKS0eK~Y-KpTh|tVlpKC4TM<45lsXYGKedZCS@x}6? z#&3jfwbI?!z_SCvS3t_L>9{Jn7ty+Olq@@maz)08OB0OZeb`ys5bH&)dxi~3wqer= zAL6UHlQ_s!@!hR}BQ_CCk`bJDNSu3ThRPO*vBh^}y^fW~oB4}}U?fuBCPej>@#eII zDN;|9?-0HcK@zHZ$S3pBs)lTx=kbWCGgF2?A3R`|nm<_?e3XFcOsAe665e)rY4OQ@ zP))v!?~;1?L?H06uk-;@pz83mD$XnJIHo3j+C)VpcHpMNy~KrE*5bJl?5_Tj<_#m7 zj&JSUN4FfNJ`cOPBt9@^V}|P-0YYE>wcLjIim4W8H|HgKb@bnM5+6pI)kKd=yFF&# z66j+y^A|+FFTo)vMa&vwaE}pNlb1dr_${TZ(hkHjG{Uss91put+oVFiXktkN=1w(v zz7_kx8@cAcjPwmw$4V0v!XL@sS?4~ZP%4>iU|uhn#YJgDj9da(s@cvgFQF1NWWAWvr?b^Lx)eW(8wNY^+Z8?P5BkHN{It?8D9_PFg6$VA%fxsv;dKP*Ts0cdhOG`AQP?r^7(cy_;TO2g-0TT)P)uEqp23Dz%=;I|CtdY8j^vT-#PFN%3&khz{ zyxuzr*S@fWwnsBk5~~h-U{M(loa}p899OQGLwq_jVQD-Y;^Wq^sGxR$kD%ItN@zO& ziI`Gmv#j1$@0k|WI$^+GD|k#Wu~QXGvs4$-;n4VM0CF2=W(C6Q)Z(OfVCl-LPux-cd-8K*p8M)YsRhi z&5O@W?}rHidbu)vf(LczAqS~A*zS>miLpr=Mr`0 z7d&L%WSu$*J)QLJEnScsLzt!)RlumtoFu%Cm4Q3L4*$1eUSnm1w zBGX=ew|V?HaIP}xaU3zJseC|F>x0Rm@XC4SLWh^j$rCP-H>L@!kDuKa3&lMnS};CI zPK0qYc;iY4G$0ri6@*!PL(piOmL+hM=KH7aRtvtVR!uRN5qIWVic(ZB{*mLXwYqRO zi=?q9J%r{=^yxLWH{Clj)LWP1)%{CI@_OcO%d=zxFXSs(hd47|*XUAG8eOCP+JmpB zRqS)E#rvXmBMUP##XHF891+b&P3i;A!#;JpBBn2k)(dKCfnvn;z-tGMmtCxJfG z8S{z0lNO-o@D7k&OS$h`nabH8X|2ugj?KzxLvrXAPMUh3o@thq^GM~sdF5LO0Dx@q z@@3GeihCUD5iwWrh*ta0<_0nu6@R36%d#B=ZKsRRiy}JV`1;UepsybZIglnIMljXU zIetZCiF$jH0O_-i2AUBn-8NV?mo+6MqXpvxrUKQbh3ZJR+s$~g(KLQC+u6TN)Y*{7 z>+37;@XF-Upxxk_L(b*>`>nW=1J9qPvdNY0Y~N~ah1gz(X%XsB>ZIoubcpVps?PRg z{@X}yss2WIvGz-MSGK!<0Qb^f>+`fl4qc#YszOI$JQCG6CULtz7WTF}J^biTGG8KB zcp6wY?}05Ds$PT!oSXo+DGgvYSa>OTzVqtoF|}sF1;ijqIP~Ody=pU@ZFNE|uI~!g zrAE;>Qrjq#Gt1WvBiN0OJD&I}O)^&4L@eUHOt*6&ppivt*bDEkR;IjaWM{SM`?c(C z7gDXa(EMrA<=#%Kw~qJ;8o*kJQ=qVe=}Rh#pt5VURx|(Eql3zc(LVOV;fIX_ll5$v zuz~B3ZI^^NqFK{7O1$!61CoAKRrUOqftR0eDfHu)r>C!{tSGeuA>2+@hhVtsW+h0E zf?Vz;XSDU-aGeF?7iCKWAT)mownaCDS)G7W=&zCTa#3BZW+2CQ8$>?bh$ScvGpYB> zjrh87QZr$c;9j&W?r-fbdXDUi{)vQR8>!=%Eo*uB8XqS$A-z%f-y|Ok1wwhTvDipF zF;{n{z}?aMcC$;@)ocUH9TtEryG?33q@=?aqKST#)f|dUdYIO&rhAkc)PgyFk%Gcd z<5hN^EF)DzCox;n#D`Q2mFchnU8YZzSaU*Gim?-Er6)op;Up+tG2 zb{=4t6GPpCLC2dKKj(GG0+!L1exiqj|G(n{HBILQIuAKxvQIj)Od4b$nv%}$+twbc z>(|o7adb=>t7!s?YSI$-_b}Cs@(+1__@6gk?+KUeEM|0d*wT9uCM^b$)Xs0L+4}(A z<6IdEE9*t!N?W)JA&C(baM`hQFu>M7v2@0&W`tX~17LHWRgV#ghko=_TX36bgj0p% z%|S@$;|}ueN5J%gcV>c0#x~9 zQjYpSYyd&7hp#vb{L;2#*1&A~xYrZZngoU<@3BPs_wLEcwu4?ET2lj6dZ+aGxa~#P zj2_2tM!Di|pTdPl&>ECH%tAa7P5o>{jjn>sRfG=P-P;?-Zw!~2u_9%pf|5^uHkw6t zc*^3$^Fj*r^HI)sxH|WuqNI4Q?;@ze!eS$XIhEbKEv{+L`A?4}`r?@xQ<|4MLQiX%kA7Czy8|xH%p8iesz7~` z8m65@>?JH3;}LP+!xY;1`1-1S5TdYO6;MUnl)1{=f$pKG%#nOjg|%OEZT0Pw3KfP#|668t`>5o?eN_ z<|KUdF`?bFA9}Kx=_DhQ#^F}BV95%szm0}UqA)=;;B7hLn=)kItr?ws9>%@9T;;wK z@aVOUp=)?B4XBcOuf1pQye^qEMZ%(|Y$hDKq-v`p5YXFh3Fn|gC!0hlDX4`?a7t`J zCp8l3YjMQ*(WJ}?9({K&(a1Ma*SU$f!rBPgC*S3m*vGQ+_@w}m$7I4ng(d| z=j{JPwUo)Q02Y8KpkZ-4mnCOB5Je_}t1|;OM6i^5(Vt(sGA)DCdvJ|crVq_avebbo z`7iFJ4JgkXogN$WH}LT`~8aFIZAHJ$*n6F9()yS8ts;Q3otsHh`N2l%$31cKxgu`h6cEoS?vvj zYE@8x8sk@>z_h?6)PWU-9>wm^00DpJPP-x1&rv(}1Fn}#6P{j+DUwJw(}-)Jq!rSD zzNkXV0TN%vxaS4VaAMAjG$&0>4BUAS+~jJfcfKHL-2dpR1>?f09WbHPf9USzQ?q{7 z;QUO(FNwP$N>-LSyhE4EjjT;&U0v1a^>QN*5R!z*PLfw^AItfI;t8#7GwMbD_%E4V z(np>gGlZx$FZW}8kQ$+ruq}xtC_T51*9?ahcHX8?NGCUV1q~8esUy?TY2Wm zw4lfO`R=NN(zjdpingJa^sYES&utSZVhEas%{Z#;75K#OceETzN<)B+pZN+RlfTnk` z8h;{W|4aS!>KVxUwpT)$2 ze-#tUuKbAs8pZyJ0iHuhR{v;@{@ozWYXNxR(34iwOX;A_UcZgm3Etf+M#o`1%@0o7 zfffAx-xd6R#xTU+s)+>k|AGN@V2`dMkc4%G!>FoE0@@4X&+`eCUgQ$bT#Lq5CJuTa)<7JwW2r7Cb#W-<4h0}Egq%#4Tj-X(Ud zo0;kTVB;i5IaT5%!J;pRGHP8G6I0Xk-XZmP_Agud)Q-ZZkv|>Z8~2$RuiQU}HweP6 zUH_(i8d>DbyR4Y1)ZA8%ip}K=A!MvkW^?%bBNlBMlBnNT7B(~QzOxyaX-GRA>M|o1 zxZA5gmotw2QqNH1QQRq>7bIWlWsE+Fcv9nhRpz_;EWxS$Phxv#DoMOHlNF8^qqrvnmAFK5}oDA;-LMW+My7xyo4nR?}7 z_AK#`>CTY$qEK`A(zbX=APcas@RtQAY~ssmbVsYa_-|wG4s6O-%b)ig5YVr)7gcj%_x5D6IuJs!bKyy{>Q&k@-5csvRNf28eh$3-{-u$O%PGL1y+*L zmnm7cF(QO=N5i!m*uRI&vl7RE(oAsTho53Wo!s#=r6by7{;j(6Mw0M?&o#!z6l zgYibvgU&ryJbUTF(VD&LmbSTXN)E+F&4(Z8ppk0^K~cPrF5)(*=`oF-b4!mr?|fsm zVLHF=^<;WR5O%Bs_|kDMA>>Fv4OtN^_|t&DD0_n?XeH5_+;f|(MBPS>!m)NrY8P(K z&7x!eeyef}?3`re8XyR;9E{R{FEc>;-hHZkUDV?QHt!#u9-NxOIp6kBRMsKLsQ5~3XFbf{tDyC$K`HbhtSMk)fknGXSH<7T)=JIraPR)N5 zD%)(E;ck?i3o$)}J7GV3K6_QYr5pILtj4c_I5|i3CKm@Mxh|p?kK}$LzhiPX=7htTnk^Q0*V9s5a_}fkyel z()}ksf^zbpv^$#VGSUl^y!K{&saSmehQ=R9U%4eWzAl3P zfX_V&)&RS4%UZWk)RPUR+Iy3h6%kt<{u}bQbO=2nuD0fS?4h>UA7ImlOC?!wXzO8o z=9fLl5_J4`lsRa?sfIT(t0pMSIAKLhhJVyd*-d=syQ#HGm-F>nlJ8S1-pnp z&ng^jmZ4^^pa^^$!xjt%jq0kAWaaXno?C+4d}Y-Sb9^JEg5S4(dy&84gB<0HK{rvBXT#xS5xyRq7Ro@t0DJI8oJ* zfQ?xgR5^SoUwQUX3KP%8);E$5BO{FZw~?8@FqcAkmEfRbWD+>>VrlQ@v*Da$GEfjY zWdt+Q0I-RGN)O5=g0>{0W)qKo2^1V8Yuz2tvky+~(l?O}Y`E?=G*N#?2Kdb;z`-g} z#;V7Eszf8B*Wjd0{q{!f-;2g^UuPK z|33>ivd_!VbE@Pt1xD1^tsXFD7XYEp|5`>ACYJmgNDBW3k|FQ^4l#BhvwZ)ha?@z` zul6O*Kiil38Gq2Fgt;CIlfdidgM}S^0%onI6!YZFEqr?3yHA-)Rk3`NjhC6EH7;0B zF}if}MS_Z1Gmkzv>__2Gnp0Jgo*>%hLAA;gSE->pFf#Kq8juJgv%g(5@V|JGJXw}=wG_@M zPFc@6-G!nFtv)STY)LW^Q#MWXm@9mu5^`!lq13ZRZ=)fi{JIjDS^~lEXDVirYM$0) zFkpdUmuhITcn5l*7#9ZfHessw#DY+uV>3#qWq92gle9CbFPSL<5- zvfsrx>KsR8#r3se1gC;864<a9XB<8Ku*m=H9 z?^rRcO36=|2|B}C%y^H{EK_qaA$Swy7NH%e9aVqOrWmY#pHm{)8r((N;qG7Z?Mb!nY*9V zIYad$SFHhnad=lvO&LN`L26R_QBW0>IPo33Un_RhT}khwIs2rd4D0KHOT~RwPd!?j zSjm?N39zHVTJ+Nr!{|~p`bI@I>uN^!`6LrQGw9K1p)Sc+$$v!t2ptvICY*dppc50g zG^+4ul}X;^h5V~tRt@b3w$CF1@2gudh$6vFWGkweUNRAqP-2Z`hSqgv^ajneDLgi6 zgE3U~F}hF0+LuaxO&@iS=69Dg{6kxvPOMIhLwyG>)Jfp%@V=BP9q&WwYK~slvlb1E z7=!C0Ey#An7@P>$jMEyp{(7u>#ODpYfGXFS4hw?u8t9^!PXJv@%pDv|z}5`WwePoH zj)wnBFHzxerj16%S;~V>yQVmM%FVkk>>rBC@QwJULZmrNW_`sfLRG=O{qT7Ol1Q)t zNSmpFM+iyiJsm=H9nOnm^;6a^89Q&5rEuIMM&XRqw6UACoMNNfJMSR?2)I{pC`atj zID9-*%`Wb7c0pfiNL@q0`RXx%sxnFDqU2O>G7&e1kOZ;au%tF6eZH4f(MGPa+x(Vc zQ1P^#pWtyiumn8>=W4v>NpP_me6;?62AF*uN|%b3m5qtiz!}|getS>t9tD_h3U2Ad zAre1qO4P--2hKO1k-C;nvgLaEFxuK#Fo+w>6{Yf*6lD{`;idp2hvap0Ide}0@P6#0PmB@u?M6SOq-i2P_@n2l23N6O z0p2cwd)|H0iki+B#=N|6k^w@iVM=`}YgLf#La>tzuV9Opj(>}7GzhslXvC^dfBx=w z5kLBJSHUl9S{|>7=)()dcqHLh=?7wv#ZY#iMOsG4e`zRe62tcP6I~@BLdKDX-3kzY1+pN!M};VAS`7nbZD8Hs!|3(T}WPOwu$e^?zwI zpi*kRYFWX5i@-jWC>Ja7$V@C&!}#87MTp>{gpiXe&;@{+^Grl$r!UHGIhaH0kjxmxSON(04Na+jlQ=m-X5B&H45r!_h!#sxDnn(y*3a!OobB+lH2O4JeBM|4^+vmCF@bd=U+TG zzXHwcuVj_%^f~ol?aX7#%S!xDOU|q*iQ8u{W&!(lTR9*%YmAHy{e&S|@dZ`~n6HDZ zg_J@Y#vVyI6&I!r0b9kPy8?fsv_{yWR2@4%N{|tF&NU7L*5D3x$^6f7nxk1|iGD}Q z$~^^AFV3!IDm`TuI;|^RzT(k_n#<7i`KuDG*Z&Do`ovKGK`8M57gxQ6kGplGI$o>^ zjE-fmyQ>*1m-JOjZ%v4F6EQjs#kUtm21~wM5)P0*@+!!$2H|XSOZ#UmJn<1PSXlbC zIr)%!5#&5j*h3IbS0atEy)&N{$+1*Xu%fpQQeU^gQwD#>yAS?OvHkD$>x;~CCfp6? zCYS?n7K|&fzrJMj*C+qweOZOyANl9|e?CEGfYj`Y!k-`c-+h0% zqY~E(H5lAbVK=g_h#NgJwZ3?2GYnu%HGzun9?TK++wK!q*n_ejK7iMb8;xLY=%v_^z8VU{I7yy zjVBl1K1p$zZ2ZLWaJvWydPRtUtcKtlog!&#Lm*>j@cA`wegST%s;OCFZv` z!_8Ze)n^KWa2l+R5B17TnSXK{k4BN(e|(XAczw%=7s9#za_)!SW}BTj+Wm$m$-nx1 zsoW*x-IwE^o~nt8Yuyi#UF2MIw4#RBVD{%QEf|aX@Jt@ebtZ3aGm_0`tl}PliuW^^~phthNQ?*zUn=9E#b4bOsYrxT1b}!O!B-wc(34fe}ny5@07= zk;)$G3ul5N`Lb|Y*_%Qmx1#YQPhT*f)4Y3MJyo|04VDwY);V!rtPdKMe6Gm(gFGe*#`abL3zom{H z9|1`mF{=7huqHvQ32bt_@AxC&c)&uvlmz>}-nv>JqgpO!qH}+>dx7!%;AZAj)HN zbU&$?*lefcJp)s*B#3!^$!bg3tz9C4o*>f+Fhx6?^sx0Kf)F7u@}=wC`|Nh zgqOMVkM;%%+V|YH30o<6@_%vno>5J1-`8l6rhrC}-W3p~h%`Z3z(NrLMF9aJC?H^r zbZLPg2uN=NN(%^xN{L9Xp(DLZl`b7gsDXr>cXQ79{omhvKi&J`jyuNvkbwn~hv(UQ zuC?cyYc8j@Ovsip#h+i7Waw#U8a_`~@EL9QxHd84TRT$1-r}*urN}|W^Y8G!(cW8m zQfCk*LtIc>Pr zQYI($$OMM&@6`l5g2*}cz$s$>s@8kA`bs6Cv%XEw%f6eXY-s$+(4phXl5Wi`8nXGa zHj%!&v0G0*M|^`}L2=5v+9R)IrWL%jroX&MYuv$Kt!=wD7~A_`8h|MV#)rCGFp4rE z@ECmh<>lcKXHmbx1uRQ^b#M2%nd@YHiLzz(*2<#uSCRB%aidqSc z?ySxZF*Nrf(8e9t0W7iz)NqF__fv{r2u6Iue1EXA&y`KgzF%q{ZJ>osuY#*)gg9l^%}wVlR~+0GvUNnjY%CQM(9)M z3m2-5cc3_j3MD-wIzFl-q(|pZJTg#A9rqsh>c5P-^Zm|mZQkgF2ClW7hPaIQdLYf;4$Q8(lci-@&os4$TjDSY;mZ!%6JM?@sp{k;k^Gax)ri!h4imK%sV-&j&b0 z@Q}rKlIr~~2DYl59oQu)+63?{*EF!cmw3aHFQc>tRc*_0?4$336=6c;kuW`iW|ptD zd~FUsnCI0yq>PML$~UMYPn#-C>>$Y$|EW4_HA(4JW{%};kC%(r&vi&p@xO7>s=y9< zSiSA?zSNG5Eya$Pn{em!?p+JaO;fynmgf^%4x5L3VK?drcrAGGQqXR7Z@8sE@@bj1 z_U{$uDWQA}63$pdox`0?3xy54*#aAsQOW}grA1xFHP4#RtQ)H>XT{Gbo%Ip8ZyAKo z+KZ_pXhpt|Q}xj-2@v1kdYagua{2s9h4gjHN!M;)39tE5h@aJjC;CxkW!Oo%yp@Xy z0T)j4NHou!N!T@M+dPGEB&iXV8pTgttnPnWN^)~PyC|7RF?S=?D}3_xHW~1LTC+bN z(rgR3F4m#>Vv@!0uEweHYl?bA$s|=L2+*Z(?~{0rw=wdcZCr%5j--CrPiFo)x%$dw zyv5h;yZJ#N?XZc1;wl}Y2R55m1I~Wxd>q>#B@4s)5!nc`6n4AdD^hmf***4L4irv? zNXKbv+)dqCx2GOJbfBLb9Gv$PIRrX^mAgi(!@psrqU0DH%Ms8(eG-EZ=|HXn?E?-@ zXqFn-JoKb#x1*0E!VAgdY+{;>P# z|NbY~fB)tGga1EJ*sB8`IK%*7g{EY={wXz#Y&W3bKio#KZMA*jU08}r*tTVb{3A(A z`X5PJ<(*Z^IndmS_c#iEf!p!E_m61uZ|kIEMY?Cm#%+AhW-EoKbZAF0(ZX7H3ROsL zaO(Fv{`HeI)T>ja8$0*tf4_K3(C+_{AC;Xa&U9Am-tB8REixQm+~6lw$)U9O8ln2* zx~!%d7thgR#23fRiV0u+C1LHj9IdxrO60YOU46Z}Jmklz$B zphA=)gdDxb!hrS4V(#;!dmpkMS$q<^z@VG7OphX&bR1?+7drwCPV#U>I^qwgpTA%} zSFvPuv)u91lyn3TmfDiU(e2KCu(h;aAg-z-V%dm13XK|g;;Q6e{vbO!@1^2;aWRhqmlov(OMz({u1DXu)PVr(r8?jgEB7X(I)GpddUMig-U3(|H8F!4_X4!BSNW-VjU%g^d+3tb!3PT}_h zB%YPvTT}QVWMLmJ@};Gcidou5?ZSS`Pe)BdOY|()^lC>=Rh6Ny7+jpvd3be|S}ex7 zZ)TSU*vFYP!OKA75C=HspM>o_KMZda;6g#?HeRZV-u|gD-I0?OWzR23TxI2CTKU*Ak@RdsFgy`L+eH)>v%{`A^2`K#}pi=Qb@L`jE1icI+>fsE&Y z8MqwqNX?CFuiePx@w+y&q$;g0kkFQay?c}lbZ_NEb4nFB zT_*NX6V+)zh$o`30mYAB&WQs zt5Ng>g~oK{ipp~Q&resbNDj$16G{w9_Nn}ZeZsa4BF>SSL741#GXL2+$&y*{LqT2N znd|jv;n&stHS>-y?$Ni%AAW($68Y!%eBkU3AvazN4c}^7$G&>m_0(Wiv|B_r{ljL2 zYuMpae}=8iaUY7dUFctsU**T;*u0_V!F7i=-XhzG)**)*>?D(Me`znzgL1nFP_mX1 zUzQrprQ9(t0&tWBc%pve%`}o6sBV<@9_^x!kKy0Q3@NjDqpJ@1hVv0#VK0YkdKC#< z@r!XJrTH=skBk1Qcfz*e=TI;40>Qf}kx~zS8cay`j2ZKIi5RJ+yko{Z_Xw)H=6SC9 zaERMDA^)5Y3@Gz;&blr$LS#WX>pEo|xAkxbOFGd4#AQoZ%-T+In;I3qH!7nnyhZ7; zN8?!djfp9RyHV;kf-{6x=lY$(n-s3i{{{;GE0pfPAX~S{VDUW`2ERqGWr8VpVOADV zs|oAwBT|=^{iqpI&k*~WlOe-Tonae8w8wyydxr}ca@2F-h{=%Ma4c~@>8OdEx7EDE zb_kzAb1$nY4hjQOV(V7}&wf4OrDFHRhiRV z_jZlg>-JWNqFnMcs4q|4s}mk-sqI>|cp3!~!ozR(fLvy`1@fwNyL?kAoOQiNdlDcLXaG6&PE?G+5uJSA2ji|#}q5RCwf+rTT;B~ zr}WsPRvqTd`5NBhSH*p+M6%|BecOnr(ldEW6Ysb;#cPZ-xW|K+T2jD?{Ve zmOx?kxIqG>5BOdT6K>i3Kj{7c`gQq=kS?OG*47Yc%OmA|9Hs7-Fuh1LT6EAB<2|9a zx)6h|4Nx+vSq9Ijj(3Hl-NRuv{M84w&GCzg?))U1XOBs9wfvU`F8x$YxX=2;NE3K5 zuKq_>{{P^s+NmR8XIG9u|Hsbmv~K~Se1@h@mEgEXiOp-Vbyj(vBt4mFE5a)Fu1%j_ z@X%+37D>9eI_(g?p?2}xmO`{_=j(lBuVOD3=tmL4V}N=(u(Fq|lMi;>4HKh>ZhTj! zZ}sgQiFKmKULfa7!@W9R+-9g$?SbSCQFWq>LB<82hmxqL(922K?mI`CBPBoN3(nme zz$zRiA5Q5yyyCsqW>^q7I+J{GXKhb^xNcBt45jxJxsQ?RI{sv?Ht@+#@b9_gq|=O) zN{|IFhvdVf0YsRhOo2c)1aQu5JD3KTKZ_@0%X~9;SrOkxOU;YK{)x=nv_p9uy_Qgj zH^~SZ4l8~~ks_r1sm?;&8)qC`)MM*oez-RXAef85&DDHPWmplp+B z1Y_{)RPaW>&A+mXIcu(GP4m$tYW$ih=a(Olbb3Y5n|mB3FM*f%R>w9)E%z^oFYW_& zX#4<_%fBZ~9u6;5Nxd!jIG7z%bJ_KrTo_dGsiNX_t_4v>fS=UKic zSpi_PLLcIRCgT}apXfl6d+h1hTJ@~7U@gYNd}T|r%h_DBqg(9i+L z$O06sDjJRj4&M2%Og^5cX?F^WO>iwKyiOmlE^wCNh?fZP=cz)j%Wob|bs9Ni}(I$3@4TUVQ_UYzDMb)-r`8k8XbnI(6JZw7b=AiTUE!J|0?E zq2v)N@w2OP>rG#!Q*F6*#K;r8$DU_WgQaXafexGB;<@)Evc|ao9naXExaPeE6^3I= zW$^6Y{{?voJvia{fKq1%I^7KGv{ey1qVt$08e+;UN%zsFUyHBp<+v$Lo-E~WoL6~w zc%RmY17AyX0A)!a5yU~&*(vnkrsk0~0S&SbdQP5V-_!a9kkyEKwzKGn21s>m{|8dKTG_sn8O39>jaD?WWl~vC1 zn92=1+37xduf4XMjh9^;ulo-CBdYhnCB$*fq4$ws=YJI2 z3Lm>?EKOn&L9wpoPJga1);WXdp+5p!wjD%oHov2C-4lVqB7@PtBg-j2u}P!ARVmIV znY2A)&?rZ=%YFWIl?;E*9cuJpp4R%I#l(%k9zW}NORYRg`+mWf$+Flv@kcvb3yn$4 zvf6p0(!A$3)%j)dQWRbKH(DwMJ zd&QN3mYU_KWs&bb(Cantt(i^p)KRt1gG{)vHs?9P^l>Lkn=4-<`-2ilj`Ls?@HiF0 z4q)Dp$r5;nUUI`fINrSnDzq=Ldv7WIa|o!Y)3!>5z^lpa2tumKa1~vWW9G8it9aX* z@lkUv1p9X4Xc(J|Pb!r<$LD{+=opax@4mznh930K}peNyrr&W_WgzV`dsUF1t|=J+4jD=D+NQBTw69O;Q~zDpZE z&`$`~XrsAua-Y|v)UN{$Rk@$zNkKZA_HW8{@*wQq9F=hdE!ef55Qperbpb@r$lrWx z9tp9QoK`Rr;+BUP`#885rt>@KOy8e;h$wIB3D6jS<*00l;&&0CKnNMJ<_L(iddhhM zZ`GNC42LA{ckG|{wptu`bY&6+1B*285Qf~aDX`hcp7WG9)4=(-5Xf--)#3($XF2Ur zG*%wBVII9}8?yD{bS3V+(EM8i|Qh%e-Ib2SwxP<+n!2;?X<+&(h`$$ZR0nYVM=2ptP7R=1PUS z|GHp4mObd#zL)uO$tCpW$3I*Pe7W>|p1m5;h2pD5V~Sz%TiY@IwgK%I8nZKJME|DbVj4lT49(rzaO%RTG_fnjp{#9i|`Sk8<1%Pr`t&9|b&O zCbAWdwZxpX30+k}CSs(J#h)M9D+TA+e%9H0|Lj{?oI8U$ zC-Up7Z$g=5S#^Gm?6vX2*e~YOE6G9KwZE!Eg<^SsYac=T@Q9ry{n#Av12-$DM4qW- zzlNuo!5T4q&yy2M-&wN#M2>r$yjb-;kux6F+|vO$fxDgm)LtSZ^oJzx^X#Y()T~sh z$3!c538YSQs%Ydr$DC8q0roepvKnaB61dt4E#8Q)8D}66wCqrFCtl#KJ5s1sN%c?4 z=f@2$Nz$L478Yb78uS4th0h|2iRJj})NvtBgbR;imO$YO-n%(-7;)U%Oh-L#F$y*>`Y%$i?4Sb93>H3q*d1KXG@@dbQ( zk`OQS#^wC!W%xcwk365iw`|;esfgUKHo&eL?3L6PQnnK&<;K*v?Tw+*c zsH4njBc*C!IlNN~3>_Rdf(ln72^aY;O?AnU%(Z`q(JC8RY*$vS>eHK%?$l`><<$vz zW*HI4+*6TRQ_o*1uz!IbvP#Ofwx@0y(fGSI013Nd+h@y`3p#AS8nxfZeDA0aP&(@h z&%&JARqnQ%#Rl*4>07IAbVs=8i9;BRc;kU(n^bV!^~8sf!0Je_=SINziw+rIoySyv z5St*XdP`1+_z5g?t7ZK@`rgr_XuCDx5=@;6vVehq#oSc*=#2;1=jAZYUBoHu(ls;h zCcgahW%(MJkrNOPiRL)3WhJWUOGU%GJd8iCtZx=RseIiWs~s2fqke&zI8sR&gq8az*N>gUTx!iE4 zSCo7kG?NDgF7alH=!(%Q_Pz)k$T(v?=a=J=MZQkBlJ%*}shAF)% znS7Wsi-~%Yu8nD+oR^4r_--)A{fFXgK#W#}q8!xn+bw2x&6+<;BiZ-wek(1DyQt7t!?ZpRx$up)%EcS{!(;>) zLj7-NE~*18L8zI#uxLK*<>q}{MLN1vhx6%ZW8T@XkQoTA0Y0}`sVtTNZQ6Di4{B!7 z@nM37{P6tM_&vr`FaUv=6KKToJR**mgsRAYoz~o!l5-s$*HO`rYI)7{Rm)H7{sZz= zqUk&WCtz>s7`$uFDOer0Xdb?%BN)oeD}C_6o8V6`+()0`H?FjoD-c=Y%O&;9LRxO2 zntge~q{l(z#u9IQsXEt6k90gHO>N>>?gd!E=_X)fJCY6pP~}{=sO8@QsCQl5X2tHD zo`j1;oSG@?xss6CS`B($yL!O+{vDq;nE_4lPk@dlV8V9~U^Zz2Mo7bX(j8(>Y0D2m z9bwyF+L93qU;Pap^yswrw&Rdhe?c%`K+Ezhwuo$wpJ;aR>r%1{nk|b$x-=o1%lhl) zMm?T}dCNk6(HovQH4g)1S7SM$?dVh;pu8GvfGe~f#1~59Kvd;SrOg#6t?rrT=Qqom zx1FE7;IXY>mG1N^O35yL2`0U!COGU~V7PRDSe`K@d?HCoB4TM#bkvFd#Thz9EU0+c zIG!MeG-3?=O*!D`aFKkr8DtsL;j(pVlv6%NvcKT8}WSLe2&oi4F0h_g7_9(9t7%Ev) zKJ|0e0lSf1Zjm>pdug&w2!fVKUPeE63`xI?BPLxe&0a1aPYc>%euLoTe-)W^+xQGC zOI=nGePE0_vZ!=dDKt>`%7D5aOCM+8;}75ZhKk5=QesbP_`l*#FxU^E_O=A=xbh%`d*c7PR)BMrpr8RD=`p7^4H;ix`z)2 z3Aq3tatQ{_ixi4iWLc(awqO~JvzCyUATQdzfj4O8 zu1g}AG%gtoT{*8{_J%$F^u(K3oeOzc)G-?Lcts%^Y6I}gMDwor2@fHmLAyTn@zUnu z;@9nh)%9VZpHrdn7>xl{NJbR07p%>S$RxR)Lqr5CP;zWLNmROB37+S`p@{WtY zCIH4jIE0Xz<~Caj0EbH*=HZ(2FGc+wx=-M=8KAXfSV z+ApjSw--|W0cIB+e21W2aJn`lhD|V&B*>|jO6W%6)CDr&#na6`y>BVZ6*a?>AA>19 z=;Drkr#Mt*Ak{?RK<%A6m!nW?cf4`kdIa<42!gqx*;)~s1-5O1pR8Y>r7(Uox|ENj zDe>pMiAM^tLX(;{4!kbf3s$ZJbziYPpoQ%epPCHuv(op)^>05~HSTvqJh-(sVDpOZ z$`10{auX?RPS3(FF~~|e@i;eNOC@d3P29f9$s+i|+vXWgB9>e0VWh4yFPMNO+Gnz= z7MJ5zecMWd`I6w?%stw#OLc<82v4mB@46Z!rGJm}Zh+*32d_Vs+J_5O+}qRRr}zy5 z)x;hRr`2_)UTP<;sS8NqHk9W#7CsL^)=z&?1Qq1G5uBYsq*8A*`8N8aXV~~iudb^H z(_Z^GT4N)zhw<@=^e{7GUG4s!ROG~5dZwV}rK-CbWr{SN>~v`A17NYW15{w13LEDM zA;v^6^d|>P{<3nCMHk#5Zf@4&^W6jn4x!YDr4>tYfd1DB)RYX+Y;h7e>rWn)WjW8M z(cQ=wQhM1pI)jnlk|joWne;rQ+OM5%88Pq&EhN$UCl+nCT8dC#I~U)(V*dOw^mT@- zqMP(RNt8^1zu#wF$|{M%8Qg;UqHs#3@mlHz$u({It}Euw0pJA|j#B(2S8>ex1=iRJ z^MZ}I{7%)Xbz@D%aT8H6Pi>#!;M|^|7ATa4G;fyH;J!Ck7qxn%%fc;y)53L1zW|Rr z?ga7s^iBLm>E~E0)5YZ9sQQUwe~1ug8{GR)2H2#0vvw!e^!#YxZakf@MbA?l28ojz zN4MpP$;B@J@ek;bd~f5y^{WNw(VF4m#r6WUJ6KMKaqpCF>yEIg%jTCE+j;61DR?R< zUyt}q$6?64FiZ5<&3eJ+nKso-=Mr+(kM}l*F6!jl5a((9pdUEjf7YU$5TJwdSMrI>og*1;U$T=-Z6~ zuAHEp0EErf-V3ht!TPRnZIedacj1@QoEvHP!*Wu8zvY$coT^&|)ULr^EV1|t`FfyE zNgr~?hjg^%H5l3P_RWcF3pQzuwcqP>jw-;mE+tD|Y#m$BKYmi8Tp4F{Dcr7Ylw{aH zHzl8_Ft4NHJ0s5LGB<`S6n+-=@Ik^<2u-v#Jk@AFWIc&Tw80nV{XMZVEm%`VA>E#O z>_3Dd9y)$={#p^9WHIOlKQmztT63Rx$+vWDS_Gw?5L9t5qdW?UKThqhi199W3jp&g~Th|D=51wfTWu#;!=%?!E*Wh#} zLzog)SuN6a)HUO<+xD5?Sx#iL!fe_AKCAQE{U=n?fKchP*?F_%{ZspJ3!*xJ^DRN@p6vOZNb>F))O-&J@4Q)@Ha=&|7xp(Znl{c^!l=J`sEn_YIl0=SWM4l1xb!-0a>hA%N6H_^4+0hkacjD%PNb|j)-jeGS`gFv)JD^_g2m-P1jmD}nv@``2k%>sw41eQkkAi0?jNujr#F0|Km@6dwD-hu)GU zhR>B!+P^ylFN6w>Rsa5@4{WwJ;4pw(D0Q%f|5>fo(-VmbSBWy}5!Mep^NBO(7Oc*# zI-!N&m-^_Z{OF0bU^QypC*)Al)y8r{!aTEH2oDH8sPv_pQrzeU>6_?=>ipufOI1o= zM5))aXS{t^*ZeqdN*~3l*yORkX_s#jWbk`Z#vHq<2#dugEmN=ej`hx4me2RHOxgPh zm6ysgzN!rNaocJP5@U#s$JAWMW|BslaYd;wr`ZZqg;*_Q?sd^alA^g_ud!v-eJ4iV zPzGR}@{S=M-j*L`NeEXjcdiPrKOEj~XbSskTw_xgYsc$U|HgGGXnRpju~la^a8i0H zyQa>&NtT=b@4$@B#LTUM(`Fz)%tDSai@&#+6Rt>OJBY;%VKwYJ`e&p;C#ybI@iQ zIX8?oG_r}ci;?-(K<_*gwyeh7x`ld(_il|?c)k}I#5Hp_8&iJ*WOeEP1-r-4d;B_M z-E0ZLV?yNx%GavsFuq>jw3hVFN<4IQb1?d4I#iSd}>?e%kh;HSM@E!?cWgaNl?kL5r7P&uenLd_9DlUy zXM5Q@7+YjH`@R^jov`8a%hp^W8~|VNoUY4wTgipuF+uNU{chq?J4)&zQ#5aAGX`Gs;}T ztApAo*@<-VnLC>lDlpv&k2&(gZUL(`Y-kP7Y%gm&P5Sx%{8BDK!=f5OuozUBdZopB z0(BWLQqc8ag7`pN=c&{3l)9po%js7;_h|X-v=Jdc4wt(EN_HCiWAWc4ie^Ht}-mENY7d@HuiOMfO34OpOQPuBy-FfF#ob2IrU|Fay%_w$C}lV8|F6IQ~6|?8bU}7phYuL2GrFIf*f=GgzTGX#X_kEWdeu`?XNu@Dir398w~ zQ-Xt%?k6vxrCHpV*oq>e>qo91-=}(koy!ie{TUPY+YsZp#sNK*QPmPX8|$sRv~=C? zTPNLGl}-4-H%9++L$!{G;D)MmS@PolM9=13Ebfz=K&p+UR%^X{t>O5BNW zjZ)|JkmMn+USZDp zQ5STHpcp-v&zy`cA!_S;r`|E(uOAuws4CFH&--u!@p*X!e}W?TJhqD)=oG-E z{i9O|Ke$H?o4rS*rUXffQph7l8TQF`AtgU{j#ye5d4xq0InO;0(AOcaY5t#OPVDC? zbB8?+j-+iZ+HB^Mc-_m5AF1MWc3Mycz69AB-KodQMDEpBGSV`iFeh1D_x)u814m+@ z->_(6H~8$7R)IYRpk(St!c$79_8sf7JSh2}yF4W5i51vr(t3ba0KzTk537e^e*Jt! zk5`ZQ*xD>?jDw;pk2&(1L@&$l+d|lq=A@tw{KDVZ&Gyoe=;xe@VQ!PuXXZ))zMZ46 zLH|Z0w_`NQ8FDN^pY(igUOk4~>E_W7kr*A16`X<43-3Uf;E{{M!}enml3(a$_%fKa#uMm`?n95l_YB2N(7h+Q^tb@W06>?M8va#YoN%W~cl^DDZAX&_$gKk->y_7DGIO`h0>*Yw6H+T(nLtKD#p#zW>HF9bmdLS!s z8a>j8ULHd%DnWohSIpruPd{UhIvCeUUR=esn@}Ffia0FY^$y<@q^a z5jsu1K)CjCWQsM(q-t9`@o=i+V$>@>Q`2n2rfJ??F5aJx>$>}7B#Y}STZsY7u6I8A%%&~oDa8D+2eWPwvFj1n*SIMo#e7r0m5 zBQG|Y4n7;)HCMc{{>cK%^qK3CF90j5Ms#B)5e>?fiSa<2g7`96awOWMBo%8-TlgVo zJnK3GjdOFB$oq7Ecj4rLLdQ8cIUd6XOaVczaH|x8TFWEr6WK22d2V@%EfU7=$+`{8-u)S)x%yBt$ z9TFsrlDEW?!1`);$4?J{x7W$Zfa2_df}m$)M^&m>39)kcJE-<3{+l(^GMCF4jl&V} z&HsY3*?%xZHj<~2!sRSBzB11DhDK(Ck@Y^+puk|A&tlyZ%$!|)yh>0Fy~o8zY4Zke z=$5cDfrlnlC_iuu73O*mw_sf+OBN8{HN0pm=1l2|JdXgCgSBQgXClJXN;~_57EkL> zt9m>7blJgJOB|fpuNC+s7sA##6euBlzSZiqULI`*9G=joDr6)sI-kQ>R@;(MILP*$ z)mfXSvnTd?yi{caqwiy*z>38k@Tq|CXSEF!vE2UPH)-r7o(A`n0^~suK@dxacOH0R z8+CyTAq%y;J?awD4vJF9JS1^M8w^y+SEUZ4(Dl8=ea~j!Z1x!AT)kPh!;6D&!AdtXqUjOEfRUx#(csj-oaT zGl`$~oFCsQD;~Qu(9F5vQqdqZAxE{h``=_EYJLtkhR_`*-}L?LkbVK9Usf5@A!#M|3Xy1TN9SYEwEy@WmC*X0clCO>$R|6GZD_|a5bG-}e^3-TjZOqX8RU(meYTE3Prf^+Vz_^SPx5J4OzB12SP}U>|_%9v8Io)p#3)k*De37|Xv2kWVo2zK` z#^>Hrh%cxtoni-hkw&EQUyu{=WQ6z}JOC){1c-N*ZVvNn$K4PPwx{gKL~NkkUgiq! z((EV-Z9g1YF0V&Sq}`mQ55D4<3mBUXrvdGj_p|Sr;@2S=P%cmm)hPpIf>uW|2b6N% zvK;R-tCoZx(HxSLEL_A@4{`c_UA(>3^!3l^SO1s{x|c3G!rOgNZ6NP?HVkS$M>0zf zDpT$fR9h0>MVX1)6|=m52P_5tf?TyZ(;P8h=sj{1-?ry6sKPPViF~zfjxkCcTq=KS ze!vj->gmm*7fgh<`GMNGFi4*7=P`#_wP`OT5~qw>F;cKDl<`^>D>0+Q%7yYO6=H-c z8NQZZ(mmHll}t?8E5=n<5ZgYF_c}`?{`^+|?N-;FR&EjNl2`4*BT<^0tl`2het^N9 zmm6r@1pDY$ZMJVaGIpwjvTDopth1@_Uo6JWo(H2=q;kM8e)wM7|uUzZ*(yw%@&^HL;mpjqR)^%CF=8MD?87vQ-F@LlBS6RgPEm;-A z)hh2hZ}NmDGi-$dPWG~G8?67%UzG-qzvNiUNr_}P1u6T99;*wZ-!89gyY6BWa$*+~ z781r5TueVi>8O1Iy=&|4JmNrz{G__Pax8M{?pKoQprVSKTvAm|P~g{TxY>m4>3Thx zOreo@_-RUe`H1l}*G?zGC)B#UI@m2;{)I&5(^vPJ>Nu$%;Y#$5hL>6eSVV@l?v(BG z-_>{@^Op6AR>q>g8nJvtF7HE}I8wjS;qk1Y@|$Ol8T84O?yj`ir{#zGH5lflgo5s; zD@%0SCsPet}WI=OrE6?wQVXN^j z<=9%Bth#_My7A^mpaFvG1Iz#`fxAEKR-(M;m<1=D@q-wRWsMzd$Khe#>{R7f(e>y# z-FI~rrW^eBk_%@%ZVUXO7^#MIO{YP8#ZVeqm(Q2_Dl!l89KTDst*?VGk7~jY$E92c zdz&*0|6@4{I{R(dkm{5puhHKqdKWk}eZhA|kwjy{+=2b{SBh#VHFpQ|}20vt6nI^Je z=HZw9CKUF8j^F;8fPK>X)8?-8zqTA@loHztAp$|D5p{vD{sC0CxI&*8Kx z^3G^yk!F%;M7z+-r(aaUveY^<{Mc4X{zgBOaFh#CA%nPPT>4YW{dXTWz)MQPU+HdM zAM)8_AUlHv`RJjvNY>L9x&(dY%U47Y+7cf1HJ?+> z&-g~mas?q-2V4RAVWQ}ZIFZKJyupT=t%dZWm zX#C_LE4$qnr%b|p4O_3@+pzI@Lu-&pLogg&P2^bbwDv+e{g7$oG9v6YU^~~2=WPk6 zFPbEJJ*xh(5ZAmbJo4GEOJT$q!!5@NNWL(hPl=cN%}?CkO7nj*9RrNMjmJ}UEa@*4fwp#E}sTqk{>j(XrzJa=P^vy1{P|(>B7SqT#fNP%FL$)# zLM^VOJSMW?;)i>&Yrwq>F^3tY>@5m}>pHM*iIldaezFflVM}aXlxIdb2t9|Ze?jJ! z!jkvqOviZ+GNOIXeV21#)5?Ijfks*%3%k43X-`Ur@;`kjE>m1GwtgM&56v;HsV@8e zrrXQ;bdBIlO(C<^9Aaz#F0OiDm(gcKY0ezc397nsdZn=;H%|M?k{sc`M3#ajjJ;@6 z+T^e&Ux;75L?twI0@(2$A}B19=O}^P9u!(p;*g~j{!hxh03+N!xp04QGmD{{Ddmw> zuqXX7}D2cb$BU(HAdMCfBO3!smH%RY4(B(Cy`K;b1{Ea3K%)(nz%m0UFfeH{e z@F6#duU&IjlY>j-RB!xJ>)uPE8hEtTbv94NUqeEJKJ$7nCz#Xv3=2Y3zjiSIDNn#M z?LPCH;26;(Flr8t&)u};PmIs!snZbB6)UzLdUGH1{9xQ4cDoc8l8bYc^gImTehnOG6g$B@;eJqjL5$+;)<^{gGyB0Wu}dKQ zX4pCiBfD<=0BR(EYCQHv2J{+H0iSZ0X!{A?g!H(U>4R$8;4qX*TYwggIWGgoFY86C>c*U?&7YQk@rI+rsuu{$7PQ#uZNi|GK zl#V#cSMHih^nP%Jj*){*m8H+OEAGsvSFTv@f}HAsXtJ`WrRjq9M~y@et4ysg7cp>{ z-`GCba%V~^3JOKS#M9KZWU{QZKPp)94B1eFfi&g+S8PvSW#n_!t|)ug=AJus1K(am zF=c*jO!e-NMdq~MW8h*_Y>fTfhQHa?RbCvJ+afzoo?eBjYZp<+ROU%LK!O-(+%(dOMW~$blNhoSqD39mT8W@B!PHoVQTtb zyln1ajBI`TOe!-$VxV0GE78*{lv*s^ezDeL?>?@olsK4eZa%5YS3Bv!JgJ#~qa%+d ziuw%Emd5uI^`_Fly)CjTdYNOM@fvmgyAM6S*Su}Hq$xJ4+Q0|(KBzMoyD<8r6~@{1 zC7s%#K1Q|oz3hR>klQ`_SQHoW{3o*9(4SrDFWq0C%W5rT7jVQCX=)22nOt{K!5=~OXH~O78K%`Y@=r7OR-BfxVw2!G*~9x;!*k=SMqE3CkAhA zk=#0`>-DIK*>l2SeBk4^-b|Y4R(};c3RhkeZpZKiFt_7}QB*A(@KUqJq4+~aQmw2p zWvsUPf@t+)F)=2-*GBK{RO0l#-8O;ix;%+x+JX<=Y)m%?FJiQL4+1F^$5XTxb%}a4 zrNHiPXRKu#EExwc|xtc+;#1RrOOmFx4u?k?mdhF17@QC z72y37`(Om>n@P=eCx7mkI~a0p76`xQYc(4 zeIa%@-H^C|6oyDo)qHduKR0I+SB-s}C@&lnZvFB`#12^#;^+fK+B2`EP8Op< zxIJVR;WAM|=y{@@e&PL(#!EE3`%6(J|7iSa*=7VvMO$EJiOf4lg+?I8cJ_`|NkVdW zG(3D=A|6n)^o1Ue#Q@P)Mx7|SuNxc{*>U9O{^02Ku|~^)SpKzqeI6F|skqGdDb3a^ zo3~R2iZ_G>UmcVq_gb~g4r(ZU@N@ocWZ~(9^N7DERBx_|FBy!v&b5WAIOaVchlO2? zI@A^LYOF$SG9bw$2xlGN?Z|1FP2tG!&GnUO*JlrL{ z@pqxqbd6G1Z4D#RQ@9!DJdT)f2j`K;&V0-0v81`E>iJ0e=F4->!v1V)Z!MT7+>ZQY zN)v(*^Uvak-@8uZ92-aVXUC+W3b%2Nz`p&Uvf}$sp}`Fsm#3v+1N4uQzW;HP@XMxH zh!+us2)(UU<;@mS+5|STcTQ9WnBFRe*J6Gf8?$PCzn2fnxwEFu0jj0~Z}=q%^v$NF zn`cJR4rcJ%oZ{!cmPow)5r{!=KyCi!$lc zf5Y~PnL+T5k!Xh90@sKJ4i1HB?JgLcX`fvzx#go{>TITAlX@auQ}%$bIdWCxx;E#L(!par(D$* z<)f0gFRJl!guzAzmif?-fCa;WDi{%gcY5|A17t~M! zEyd)`hY)}6?L~BG`y3Z8-UW$^Ydw=1Zn<%*RHJxABC+}?*GD}2?G&s6bh<)cn(4PzFl%Gla*kgJf#*D%T z)8uNrsP1mA28{XMv(C0Xy8cB`B1e;*Lb^dWYoa8X6}}rUgT&)n@8E$#;6)q*hUYeM z>t!-BR)+Ra;2$+!LFh$L$Uj=-X4+)ii*mY{#XZS3EW`OL^KM^^>2E%WlqCj>eM*<*b(f(Go3fXqig{5Neibj(>?aAtJ6?SY;uh!0fY|MM zdB`hqRrq6!((-YJ4`wf3w>=yVUbRRZsFQ$ zLs;5-r9Nfe48u@F%Nra32=6uwf>STRO5Y%dk~Nh%V9oIqvpB)UFHzYJ`l|fDziSM? zuhAP6>@MfR&Bq+KdzS;hwm&Jtuov9Z78sLnPRH2kUifx+4oPJdR6i68D*Ze z3LxwR352Vn1()QLK66v`vsheQ%jCH{gP#@Lb%{u4oMS};+0bV_N&S}g95m~<3Y$FQ z9oFr#FMM{g$ETtMk;a_! zd_`VIMF@Hu4}|YA@O=z*ofUA+1hK>xyX(hc=H5w?T#6-&;m4UKj6ZhlK9!^uN+n<|ekw0%d(0`?<;3*nz5(;+!ER|LAZ> zH~p~%QgXF)tETz0Hp#gfGE zvO!}0JC5A#nf;p=+`5Uq-8$T2VpfA>PX!kk&XoUj!#;gD_yHIR%U-+OrSt7s^tfr$ zU*)=t06A)R!FD;fX)=7J`f7PInlb)8ty|*Yvlz%L&ObekK0uGrZ9y{NTH8*|>lQz# zj?xK@^_Y5k8PgVYEfOq<9-}zj$&ZnMz7ocO5g>#)#-9N(6u^CeeFFc^i~i7Rs0hXR ze6};(?-c%(%cu48e73JWlB;?)=FHISf528a@=P6Xi*X6V!>6F(BFiJfJ6F|`9#MHn zY;UrE+`Jn+1gOC45VBuPE3oqMU`Yfgj(Kig`AG`4W8I9LWk;O4Fz-pG_<=28X@$>Y z!*OZUZTd=)^j2zA7D+AhjzNJNQ$Y)d3!%G$yD)Nx9VAXybEbfU$_uO-8x)|n8vSL^2 z0MNTUIrc-dFDX>a3?U=;Jt5pxW4)_Iz3 ze(r0b;=+;g#QNtV!)~#Ye5XY7@x}ifEdS}_uM+rtqhXhD>?Y$*3!1B?Rbsi$8V1gx zf2bM{1$HiBBtNFXHE8K|e!g%D-s$rsI^JPAklt>`AliYuwe@b;(uemIq*DD}Or38N ziw(l7B1xKRa zN7lq&*tJUUWyK@RI9`IgZSRy7wSRXScnFM7!Y{4( zOjyv)$C}QULuBnJ^yfxC#1PU0_%wlj?9$de!rCRA>nhr8x8yr6Wx}YUPuqpF?3S&J z+#CAbAUVUIPxQ!|18?9+F&9SKOkNwv3OY!Awy`VDPLVWb%m@vCuCkW6Ws10@2pFf( z$aHFKNgk(Ir*2&|7WlKX+=}{zy>1{Nz=B=DA}c}o{%-rYoQB z*^}6jmlikDM4W6E3!W0Ao~O}2u0)V<{n~|<{4Q$CVy5nM)xx4DdiWD=-26}yq^pvC z#tb9F*%)(a!sT-0zg`bQlo232R^gZJVoL`Rs_-9x7>}Tqer&s!Row<+93!EUwL1T#o$R)Zx8D zS~fzgAo$xI43N*a*J9Haz!bFk7F(Ggn^Vm5AF<-2VNdnslAfwN2T+{6A+QlQDIGp2 z1oI-lP@n~=%!sfEFb!Qc^WRpvQDLUkV_*3(xbN%TwJ#d_ZrhIA4*RfR7^A%Sa^&9O z4zc<}n$?MuJ)6Yu1)ruGpI?UzxYGD-j8_$bxjtsXlvLFvjNpS>J1gd4`zSW8C)>8h z);7w+e%G9|ScWD08PA9jUib-y|qafZV0m z@UExdMdu)knyjh*0<~a^BYT>%0l7F);i?r$H(oT1_fU?vJiK zE@{1<9yM~%^YG4w_H3Shz>1XBuqnkkQ`a}%-*92ne<^2W&EWi0p=+jwYk8~@w3uH@ zw*Z_*cpZf!0;N2INlHqk$e<%SV+0c2j(xoEY#CS`8(#+B+Q zVC@|JG%_WbFb0pna&gYI`bviRzk9VVg=IczjZrtQY z+HzZUP=JNC35<$pJ$lC7#vRReITX%hyQG<}J#BiVR-&PDaoK;DBXK{r{=gi_G*H{- zA%==xfrcumV!^m>9bBs-^K5ZEx=8p+Qb5{a!Pn2xl0pMJl#py1O#x!2lkA0pAJ6X* z1B3E&Zk!>8QX3b)*!tc6Q7>`3z&ke=QDKIo4u6gqE?^-n)8!t3`+b41R zvyF`aUy@;xu0I5qh1IL5&RSM`Cvi)gPYOLFKk!|v!R3P0`Cx6iy4-grafumk%Q|(@E%_uJJ^yB& z4bBdN4s&nLCfDVNw#zBeyy?c4@y8GRCMdrN6u$zsX-tvRYHfA&+?YVRpV3uywiFx7 zA&6}H`TY^5uf_HJBKGI=p|*?qu4F&1HCzD2W8oc= z&=bQU?h>cMJ&d@1tr4H2-C+%{-`%9>em?6bnL!Cf!P{V>8+Wh%7Gm|;PZav1DB)VU zXTPBC@Mqt{I!f*9vsGfIcT2(*bFZ^X5S_GR%*U1nmu#bRYQr_GWIwhATj}Q(X!PZ0 zwgDma!a;JA8FO5LMpape@mfH2NGCnB?Em^5nvQbnHI0HxF4n!zez&tw^H%%nx0_+_yjQRT zG>Y%`Ndv#k8&AWgx*ZWt)cxfd@{f5uojtnbxs(gRGRljB^TinG1q9I{FcL%usHT3g zQvj57K2o=JAD*jNHsy&)mXX2~>me@p70T?ABj1tpB&d-iN7>Jclv>lN+h?{oGt=^$ zhGpX9h1{~cTR_<}WY~M0RSpd@-?RlamWPw#n&oP}o^xOzriV+{yqu^bu#35@ALA_- zxtO>RGTtk6t7-3zIRhrJ#3yIHmc*a*Kp?$DosG1bGGjmb4(r~Q z`Q#sA?2q{8xT0LVSb3S#-MN6?-WK==ySU+}ZZ&V{f-b5;{Q3RzclaF@)|z!0+U-fo zdag!c)$-zgrAt``Puxs;1-Vw=Nnjh)`g1P*&s{)T`~cD-TgdSEl4~J zyOh6|Yi91o%^ZPL*nNcjjIwal4#%w4c=NoS`jT#8AHo?tWz?lVD|WY}v!n=$vx7na zY%T`sYm@;P9aQySYl@HNV!@!W;UBW?dOw+g8X;FQT`juw%G4KSJ)#8f&`f`?ZKAC| z$_9*vDR@iVU=CiquC%1Kta?;Z@a;k4AOp!Hvj&Y(jC+qV@zCM2qTBx;mPRBDU||(o zPzGFJr^zs%y^ss_ z$ZD!)w9#ot1SMkv{Y;IWLnl_a$HVOG!qa)K~9VKJVJFX0hf? zP@!xt%!)Qw8d2o*bv!dQ(7ZO!1IXHE!!;z!3S^LsEG@|Pbz~Z9vG6eKlnIGK-%!G( z9}PxuGkWTB_$nT>F%zucN)h#MKc?=0d^gGnKA_H#*CPv1g^zok8QCSy^Z^|kR0^p( zZ(=c%cCoKTwM^Ge`nDn?Jm9Ayk8koB6-X1-_(<~!ttHhDc0#=g7s(%G`toqU3<}`? zm(9Q-+9_we7f6TuLMdaF=~L!Z(vkOmAq0dd$&-HAcdrlvHM%WsfWYz7`29+w zf;g0;)mJD!@r{IQA^MzU_^?P$GirIbP*^C(qb6yJDdzUk73F@z(Rzg(htvzNbTvni z0+i*NZZDH=ai#HOxZ4pQXtzpLU2eQOB3!tb6cFWX`2>X1J|7wg(;;-H#!&`i7~zah zQ|uXgNc+Kl;@KU3&@bm%o@@O613ea&Z^l2C^?4(u>PA9ovpt*a%JYTb#CAW>kkO3_ zWJfC$#ipQgMXKC9;iILV22UR))~Arz%F~?N5ihX)u_7;W79w2o^_Q#W4^cZ1?*L_y{Ko6V zRiFPT{e+heBi8@8MmObx!p?Do0EIsrR=k#X=xtCoF>mrBb55WAx-S(AmDe&O(T0TK z0D<@Xn*^t*X?^LBpQ4iNb{rj5+F~7iog`trJhn1}8zx*t=edWXOupYc@7E4HA9ggj zQ8CnqJQXV;q74!g(-Gf_ISqpPevbZoa_i6Z)<3VB8;$%pE>`m!aI@^($aZ5B!Yvg} zmDLKo(vlgp&0c(+tsy-3M*^Qza$()2jgPrYL6V>n_*;2pA2k(E&@hZb?Tmv-&dS+Y1R}+D$68IddeMuh~AUqiFSQN0*BiavEW~5p;?Cu&= ziR(4M42DTKo(NUA(A!}R6PMwaFDG_vfkuRm9dXX*n)AyYjhDwo!%||GIIj`5zrCQY zn9^9G$%m}gQh>|<)KFJdQkK4p^)1?YagVRGr}-%(RnniaJ_eibo2Y$Cz;r4GcQ-qI z6|rcGyG)3nCI-@JfQ-ksE>ZOO!T8~Pg}roTZCK2QF6R%)FjR&L7Nso`}X$-wrF zp!^AK{WU>3xLDAQ)LA0aIi?vicmlUEzxk6o1MT3zcXGjt?;-* zb{~g$lr9!LwtGEr^fH8=kAyxuIB_D#pURK$0gYHFPk&cvzr5_ns}(JbD&IlDHMs?P zKGkeAlVt)M=bcM`L9SuYC_=*y;xNImuLe0R$^n-J$Lg4RerSXjJM3!5#MLSGm%OG< zEbKBo&)s4ynSLYCJ?2obJOkkLfVcwi3r`|bEH98*5^Ph#(!$P}D-ONfQPlWLIdDZZ zZrGDAxz|=qTOybB8e;D@geJpP@O$GMTiL4XSNkGPB1Pl{Z_{d_`Ywn9H=R#I9z63Vi)@rcOl5 zAX@aChK-Cs)=7~3Wi&fIKiTSIE$_oT!jp8z;(5=e9|_&49?75SH%9P+JDoWpy3Ri# zCT=0t$B`ce4|SK2EiO>FmZ|hu?5~R7#RhT(YEKhB4e0HXFm(?;2p#!0n2Bh;fm0%c zD)NAg`2(Q$wGhBVv<96EbJo8gRaxgZQyL7) zkdoC^HsR8W!KGQd_$g6P+yCr7XJTh&H>H~vBj zZx?>X7+`+{8H@pxLQOmjZ1sYybLL+s4Y02| z5`zqOBA<#Xz#gT~%^{1kn|QtuM}D>bE@INOIq4zqmv6}XF>Qs868CuqRo-_m&kT`8 z6S`3Q#^C6>HLwiW9mv|PKE=y;CiGxygU%~;ZBQm{F!UNDIgQ%#znXoA40?=51F2qC zUq|J7=JZump8fcP5EbfQQ)<6;lw~YvWR%`PT-Z){|5kyW2P&|0;9c)(ooZ`i_Fj8A z;$C`!OB_#4rsoZz?3$ek&VbP9LdFc+mBQJeZ65@UyavnfNft0r_m`0-5^dIpkjrpb z0@Wz@@sV03eA=}V5C^!={5ZW~|L&NQZEGN!JYO zj)>{mBvF+&@6{kBDZzEvpXjIqBICd6s`L1pq9Iv_`cMM3uo5o%Xg;`+7%dvMb40mE z;t*BTEAA%_V~)X-MV5a!t0a9bH4f+uvKhqbkP~t280bsp|Cbv&ED-LA1_R`J z>E54~Upc+zQbR0tM1*-FsJ3PWwh zWS!pG8ZHpfC>fT&vG`S3=svToAKS4lXqKyBS|mM;Nix|Ob{N*OE(~oO#PYErvo9D zPjb+WY$W>$MC!(W9<9%`R%xTr|gNgkzTYSPEIEx(ZC=rqs zytB|ffq;UN-3gwHZ6&-zgLV^)8n3W`q}0vcNa+aBPH5TKb=~u*ruA8XsL|YlumH)& zW%HuH`OKbx&%E}B&x{NN7W8;lh0)>mZ;mOLx)dXQqn?gSiz-%p!6Qk!---(sQ=0}U z;fmwjh9fHtji|{|Ru^OTE)v>ysqYI3uA1E)${jk&3I@@w<9M^hd8jT2+iuO5jS-Ey zpCZNjQZF`%6mDi_VJMrYVppV6xNiIuasKi=v7KrAk*^Kw$UKeEy| zChAo9a+ycsn4bKnrVOhWf$CNzIb{|1;0*Q+|Tqb+c#bn!>Hezy;&*y)=g@X zZOrTKm95Znj7d4fuoF=^kB^(<1|rOU;A|2LvoIvWJYLj|An>c&cp=NDXHAlLYb&gc zrMSNOTmTk8N@>~9f$ygwx+k~0P+L`m`SfkX;#)i?2rBWki1`W7GVw3!43@tQZLLDT z9dqkkoLIQt)gPP9+1`r?@aeA`PlwxOx5^=6ScXcn?ZOiF@baeAAkX1d=H2u*c|A0R3;O(F5Gi$PEWn|)oa z7Wx_l%O`ktzytjtdJcRT<6{YQaBjt$qwJ3`{&qgjXTL|5M}aniQIM~peh-aD%<*!J zL23&s3`S8XZ|V4IagPC*m})EAhWy%l{$8y|jejco(oMGzUy{&q04quUH>=q#i|ty{ z{G$3B{&4l^rz2l(uD2U;=6_O0mC zF!0mkDD$cL-r($Tax3~YX-mGuAu2txVb6c_lAq6xoWu_}HAcLdx1-%@=&i*igX$J*3L#w08XWPa1llYY{apedcNG=A7q-)1BInodEb_<$o8P7%u ztSM+BixD>QuVwdwydD}FU^r4RK)nR3QS`}8ptr`T=0(#AFx)AwQ$o+OMyh*PJQ}heAI_jn))r4GPcN@~{_SO5;tyAAJ@uta-1lI4PcfaX zu*vxjW^qaZJ&DmxxQ`46|NrJmbcT+qUVRU_d?6)P@W^&roFCY=r!j9K(Xt7#o;w6nGm4 z>rPyr_bRC@VSoFrOEG7vjv9uw!Tf5>_h8OQkxp?~QN2@b)c0oMoqXtynM*Swh)ZxN zkY}TIJTf*f?@Q*KFI^A6y(tT2@JO-8jxyJ)GsAu%6x)$!lDbtKyO*jBAJ45%JpNQk zfw|-H<6~Lnmz(zJed{`t90}}>j*+`ds_8eZ=p-IQe!ZnTM?!Vxv6CfPd7j?L)9bS;hwRaHsZxu_SD&Li-(saDC;=OV!))+-=`IMB-IOD^N7&DZ3+jE_Ys+My6lupg0(H8=nhOd>(FmCZl%yQ=aW-;cN z?-brnsu*p#z2DKt_>!zqD$b60X#L+);Proh3hs|QPSK9l+O=Qtvq>`a@v-RZ1G1$4 z>F3(WNWA))gzp@u_mRZA(#8>d%ZqBqu*lj>wI>;&6X$Y6WEpVIbR$JUoXXCM@66^{ zmsSZ$f!6odxHfudiR7s|j2nLAJhR^Hi|?7*FAzIRjVCt7Z_hEuS-;D*n-I%}V3K4r zgSZ+5x7q%eCMXVy%}F;7l<|$SPMo!1-e9rmQazKmB_6U$j@NhWA-dW8HbJF}x8jvC zW^chT3jY{$@t}>1-O;q#93wLrJzx{LhqfjDvY{^LZ?2{2hMlHs z^46*rXp|J(zC!knuIZ4K?_3j!K#GACAqqh6H939R9O>jg|21O%AaTEh{%ai(L4=m$ zG}-(^Iz;f`&GO6revC_dhdq~?nU*JMHy>n$Xz)1%;!knTiA-n|2^^iI-lK)4pU0Nt_AlkN?VX8_A&_mZrM1m8WUF|v@f3|xjkuXp?T7^>CoBwEw2g# zHo{7XdYsDF>7cGQE-O7huC{0_H%H(Amso1Mf5s_KS;LP^@ANz7UQvw}rlh)V8Bb79 zgsl?G6s}jReq6EB`h~q~s_4}R{FWu5q!bEnF+(J<%JNd_w%M!9iG4S6)wWn&-Wm1Z z;)Cilc^q=RJD7d-;Y=IPISvm%Zq%-UPq+VE!Ti`%Yf1iQUA1dxXm5i0kFDf*7I0@-V)9EcI$9P%;@O8R*j+6N@uqNopWxG=ghcIGe#p)WPBla76p2%(=I%DPIS-g}x`~tH0b# zlc9nSX>ip;yZjCN#I^YjG$p`lb+ymVB=v!*Dc#2~m%x4)MFk>a%DGA@(mKqgLPbN}QXsLmkP?gh zjg_u-^Aozt64uAfWSo)3so1h~V?_yv8+>uVOt1EH89lAv^8~UNNxI8sgUu*#>~F6X zXyX(#!d#vd+AcpK?b=^8HZkC&B^ps|{8e#w4-UnM#>l(R@t{XcpLKfC^Zn@MiTZ4% zmqUVX3eFk`Qq+VbK}eE^D9(j&rV^>#RQ0^zyCrMpn#k298NY9NousS(5FX%kCV{`3 zw8WP_*1|H`QXako%*_Qr?x;7M%48TOi@*!#tS`AJohc5m>q0Gv0fU4}6yD`Qy7VtfIGzu8;A-O&m8$aU~(G2b8 zJhv=hM;o0n5f-;aIg!FDXC z4$FghD^*Ii&7WY&<{t!yl@PQoV&?r!N+KuPp{Y<*uEy&n_YHS-lZkkISkK(|OmQ4PA`iwwEZt=V0Y<3w9$MSn&dRsjj`6L9*cU4jZVQ>K z4Hnh&ZwRy>I$=;RvkPO3!n-0DB-~|scGYQ!u>5YBli&b#diog4m#&M`f~$k)U9(jE zM)i&`inX)&(jR+}D&QlGGXffn7KSx}fN66{5*2YQTbaLfhbL}z_qGdX`2)U|5O%*8 zo#wZdx!D}udliwdB-=oLQ)hc$ju{N8Y6=v2Dv^W07u^W;wE{^E`&Fxi%<G2d1C zviCE$l*zskXSOQc1s<#aUBi21(H#|J(cmayV?rc0yFCM%O9z()t3}k*{ntT|Ey*_n z(;?LI5A!#ENVfm(e_hf}zbw-33-rj*G4ssu06uW`K&NcoGQ)lqsI63h$QCR}pU)lW@;r@^n?vm*EK=8m}XC z%04{rY^(9oD6(==92?eH&Emc)Zt1W#x`0vUe))$Jap1G#24EIioVgiP8;$*REhAOq zpkJ~^@WxGL%DNwG*~kpkqNDFIA^hA0v=8U;j-a39RCM!H8%493IMp)+ztv5WN+I@j zyD3aD4Km&f+_f7&Vx-%6vCpb8Z+e|iw<(0jyyJNQM?$V^fbvDzMq(GRriA04VgTxr zR%Z%wY!v`=PZ@O3nE>l8X`f4|HS7Mxe_>P33{`JO7)@FceYQ;!w&`yvz_c%7N4(`S zDX?@oN?sG`$R-^rdBcm{cc*cjBbBfI>+~;j>hodtPrEW5I&hCp1dF^!z|xL^5nL5v zNiid@>0&Fc%62meLIJHZ|Iz@(-WUd7YVq2*B%-uSqS+P;D^GkamZ~$?KoB}#8f>w{!+WWn{s!O2hN)%E z?g*zWa|@)GYP5?_EC#P3W!CM8dm`FhxG#`n0;@WUrV+8P7SZu-mm6>3T(Y-JkZ)az zcaF6z&AuDmezsicmW4+4x9j&YUeJ_*c=ZRyz@a0eo-VHmY(IVp=T_8*Pah|A-Rjc5 z4T_xL>>qoo52lr&75N?41rol6F+zbu| zjUK7{PWg7rF$k|GT#(}B1@3kj%OFNceZ)(_{XoBTl*jDjM z>Og&T*YjLwPTO(X_I58-C{>~0kdm$N&9lci)Z&SduvMlUPzVDS1JSo=jN_fN^~gRG zMzR>{jlaMj)cllivRHfY<43qcfz7H^LRa)k4O#L4*>kcQf$pL}Vc*lKr5iaIo8o@< z2ul7|*(4g*Co6Q@zLJxHW3^9@vf8(%&psIQmZHL$E8fSam?d6qgns2s98d74%VI09 zO5fZR3^1}vU$Uj^WOOk87}cal#+Ted3JFa2<$e0W9BmU?Fe_y5hIm0b`iAAo)7)62 z;3OTfSB?%fdlBB3WetPmCG%lzFY3!jgtODdR3^6Q3#j5~$Q1-X{lkv}?VmG@X$P29 zVwC(S{a|eOaGFz5x2&BhBm7nqL|Y0foaBeBEI3%hCJZAF=ii*-Q?aGZOe^ztRo>h} zB?%XQE1Zx7=#dHwEOW<~&#F!Bz#o2{IT7LRu1U*{)5RKVuCSd`m!uU76|K4wYqHf4 zf6?d{wLi5kPc^I(;f-eLFrL67M>(aW-sn`ssYGbJGxDb;g&6(YsUlw3 ziw4|9c_)E3g+O*CT9`eyGBs&Rl{BM>o+P#>GziE;CU5No8_$v9T9*q%neE+smjut& zV_Vd)$H@{UvL#b0`mdnopKCy0!{jL;5u?=hW7E5IT2m5#d^>_WP&zS2Q%%ZEe8&33V`y^n));8N!zg+7Dm;linHs8%Y22yB^@2CGY2D1DyUM z=#i~WMN|K8wEfY)!im38Q{!N7*2^t52;4&x@})jFyi7S-sjx`u+TOoLNqnrV1L@>j zlecU}Ejr--69uR7j=w+@G`o2Mcgt~D9n;JT`>_LT4u6Ka-uI%f2|Awk!P7n<^9 zD(7ZkcJzUhsBYb_3ibeMsX6Try%BWnRE{a$3PIV>QBGM8ul%W*TSQ-?_;@@?f*ksv z3|sh0e7J6HR;qr1SM(DF5tH+O7!p8VbC9j7EuMDTXluOoRo=VpG1q3>8?hJEd3pGV zdpWFC1Y19GZ+tWxU^Z$(1ryR_K)3=AgLt zUM1 zgbJN!#yg0RP%9*V!V@IeZr;ck5ri2 z3%fi|OVP+>;T7OH3nAx;Bj<@;4`2Mhd}V? zi<%dLl+^wk?hur2o9ahKz?Hr_UOppqTN3MvG$&W6>5$qRLNRp&P3RlcHoG(-RGEuX zbjQ(~A0gP{%5uz^ht%Nn?wTBHU|5Lnv}@#b9A}f*Rx4JDlS#H zxcT%M=n{56k0;a%06z&k;;}%!z~QaMcoqj)k>qUQLvMO^zx>RFwg=xeT(fVSs4Od> z0}RxH&kQL)+6+_Mw%R-9ck^i4hOBg9Lh&kEF$>G`hkr!mFrfsksu-nhQ7?c~o79-P z;7;D0V*s|kmlcP&ynwT0&Z6@78TFvL{JFiU+_oIFmc7$|N?dBkla-H&bta*<6FzR|1V_#pbV_I-I@37@myDgLQiQFCIP_j1t)3;EWKMGnfII(BC5+j0R@D&$wd9KG@N0u1K}>d8~a z9Ryku1pt3o2#mQXkbgb>V_%AJ{@g|k%Kr6j&xz*f(#ne-CDolQ$idpvP^>VpokYX~ z$UhcI2z66tK0-`2wSRlondkdLpw#f{4V&8Q0}WAn`K*Wio-j%!yI+|27-0!v-FwUz z(c2+Lwqk^XAJ3mW_gZZfR}dUh1OjE&{}CuVy>{^rFt^YshF7n|*)K*O-Pz&7t}el@ z&5=5N`oJppA%lu|s-By`Hhe}1lXJmSyAZC{BBosk_y0@`OellpsYC*t?~r#fwY^BL z!yGbNukGe2966k%-WVB(=D9zom9=#8D$Muy_>R4+NFH+|V_5bayo`T?%23{lgjeP} zs;m?LVXa;C6bq#3hws6}^vI^?FCy&7>k&(+ZR&e!gn`<0sQ}w(%bPWN#_vim%CM46 z0N1o=RQbz58cgh5Sl;?QJx4M6CFA_tL<0JZq<0{6{l2!_1>hc{?gD;>kJ9N?tybJQ z_&c>d>j;N9q*LY^&kG#8Xr?u#@VJ$(*S=_^?jmH2_QP>(a z1TgR_im=ONEe6vb&C*3#=NB5}7JNEziu{}7wND4$dhXn1iESZC=!s|H;c~sD@}jc= z&TD>r#)9ZJ03@Lw4^XF4Y>+dc9{hLFMwP{ci_k?^2KyB{GWZ#If2$evSSR}?5REHX z^4TF>LNdP)ujCv*9$UAa4`9}Hf;?_O8grJ)gQL=GT>Getf$qZlL{7jMP&_@xXVc)Wn8ksDYNcJCpY z91jhGC|WS%QQLU-MFn-(=$K(esd8T;RT+KrQ}ti(sj&Ud<0*s@&^x;)|L_n5nc>K< z#GOiuMJ}jyebBjW0?q6t!C#7`c!$s>r`=Ea&+H>~MvHRJ?ta!>oZOG?w5vUa3uBkVv!>MD-?sM z6IlqIeoW9xNd+7MJJcck=-?a@n1K^)^9~e2SAHzsD7ZQ`*SUJL>sF5nPv{{}^fXYR z|Dd!Ig@X~Q!EMROMLzi7lM?^My8SajL6ybqIM&F5p35n6xNy(Wn(kejrI7Nr5{F@* zK4C01L~S20;QUxg#6<(ZJ%Y1}iXe4yU|{EdS?A=2Yo=b=&{_f++&yARr^F}MYy&n~ z`}z@kWQdlvCM_+y05J(O(+(2`x(x=H`5l%C%^ryLu7aISh+wy(EZiz|@r^w;VI%+1 zD3Y;<*LMFWFG);F^O{15c@>N^a5`&W);ECA3_vCzbOmxzs|jG24CR6M3hoCqcEf6= z_E;YH6+qg?o6dY-V7^ZRC%B~j-ab6T+m58;G01#?A|-Rg6YBng?3`qiC{_Hu(Y@#Y z-sphSii&ZC>&S+cPq~d8vESmmU6^ShVYPha$DP5KhD{ZRsoS9I%ZrFaZ8a+$KF_9! zXhT)#m5jSMdB#%<&&G(cEM*2Hvv-k^>`=H3xD9Ik#QB^($d8hLk#ARvifDgrxL;KM zxYgDAs^RmmEKD5q0Gr}0gYh~b_8CCTEH1+Ldn!>Vt$g>DBeY5(P8F^bAAVwYZWjpJjo1xNYiS_Ty3V5L9a`kP9z} zMg}=Mi@+7Ewmf9R?DG@4HXEaLl^ttl&nUu&KJei@!1{InWXs*XxP$y zBkF2iBZ=W~ztn{mi9l26Uf!_(>~MGQi8!=O#oK0oB^jqO7pysO>(a`CyN=)*EJkVn zX}u-rUI!r0*?x6T1Hxq+Y!AAo%$fK$-$@+`gXzOdN!3D;?lYSfeG9}X9cnOGlri1hXvQ3rZ&Dh81P?9KTrY)b_C*J z4J6_POT+R<${fIyJ%Wo3hoBzzb?*KgkaE#D&x7wZ&70~&bKH?65UGJMwZre>tQX@W zsR0$H)c#s!SDf9mjx$89kryY%4A{>ubEEqafReE`s0SiO2U6n~o;!Q9k->%9Y~u$Y zbz1OMsZQ|{e}dhWU9DQrhF%7H(gEiKfO-FT!bM4cvY`4)0A#vgFSbG8ETwG~cX>Bk zRiA(UMM0T^XsQ+n=XZIzOs*|rzkPc)*PB1n%C~D}RaJev5Q<(@^i9c8gk1cDyD>hx{ z)(^*=$DT=pjo^H603M@duU~=Fh2TIA@@J36p%Nt2IxG!ao?XBUwyi;onUE^;JSc$2#^3jK=l94RXi{pXtfP|v> zI&kFXZV!QR&b?7^&R1|;h=jZoa-iek7j z0t!k9xlTM&f8!HjFn&qlj>L}M%v{>rd(_i9SADkG{YkebG9LyE#?@11`);Yf(Zw(& zq_cd03q@GAAtql8(KT~!9q0hBRlplGM*Kq?g!n=6;)bn*Cx@d5@VJ4hk8IsUH|%n& zx4$Q``hJWqaq%J#Z_-@uvE&eNY!sO3>|q{`gaHLkOc;`)2Tm!08ips?Fs0(7$R&69l{Qwt4IpSiBO$ zw*W5y(yssSOUwTM?sU@6NpHb$)pxHGW7v03h=EUJk!SBL1%djN@CQvhqgl!%lDoIW z+M^aA7d@s~i`amTx99SLOH^YDs7oI+L3}&0(%ayfRGFS103RAVUVM3$t^4KW21cn<@Aqv{uFGPAsL5oH{EQ_@IDaLQ98B_P#RJ9a&A zp+|NZ16|At59FbIyd-xm$?@fhM+@Ouur4}<)38y&@y-8ir1U=(z0@dP?Iq=uoz- z8yU!ieu5psWf_NeIEINbT63x0q!E8j5}dhKOe4#N(_SzJBkIkIB5q(k9`OzH8YKcg^jKn#kTwa4ZN0DarYdh4r(SsKb~cvQof>JH^=pRTy~$?h^Re zj;&hsV5DajqaDT}o%GPKl0J#qX#WnU3K?jB5Fz{ixB~Vg_U{n{0715X{`A2i(M}Zt zT#|{|Ll^02;l4yUwGKJI+-ISo;ZLqaaH{ZNi}pz2>gd$i;$0{B?V@|HJe)%`ukO!D zk?Tl9HZQR@@!7JnZVDdQR`?^gE8Q^Tt$^eNu4vfaj(G2nP4w-0Pi2R6#)vFwwiUL`M_bAw! z?%cElPG65D(u#6(-cqrOI_GG+jNE9s;>GEy+SBXA?Wo1{MnD4_{E`DIiCFV=xWK2Z zOW-z0cupg>bp^W4MCManaBCSygOW8sg9G&^jQl5S)#6^E4_X@`<<+6B-Zl)b(|IZC zqP+gpof#kjf6a!b%fI@BC)H)y3^gB(1==cqLGo1WZ1#sAPz9Gzgl1-Q(WsL}592gH z;~8LVQ>aiE!gTRx@#%I?>xDRNoqa;oQl4o7A#m>qykx4)J0% zT)p2Yis2}&PjWmh@!^lU>`tS*Y`%ELi)%Hn!Np1?`h7Pun|L;*DcI?DC7J5|l!`#y zmmpbaG_1_L3W-~LItG_3*!Vp7D6M!_baq(dx4ZV&i5QBt1Ya32j^WmZKHe@^A}TVj zY$)2ez2V5?7;3dI->xpB{4=b3BguEb)h!f6eD->mCGGj>K)%?c59^=G8qx>Iv(s|! zM@w5j3ggXOIp!O`y*S8)%TXH_s-RTx&=YUwu{O^4qpPFs(IE$`!<{E#*gIco7+ukD zZB2RDv+iU^X!ok$Z`wO*-Ubls2$|*$m^h;4=@RsmW^*3LDu4rE=`W^(%#X|SbkNCb zdu>@RmPIY3-_4jvL(cPF44zFJ9c=Z(^Fb337U?(8P~hA@72@VI*gZ!fkAY#{PQK_uiW1&WTd+@_;1>FE z@2Db7Q4vi4EA22Z7_c{CQz1G@muchu5-13+<8i(i6fHdMCNh znFR18uY!HJPS}qy1h$7A8G9_)?K8p$5CM!z0!Pz%hyCGm9GCX|x`h>|y%vAsxJq=d zJSXLhRa!-WJ=kKjAaYFW@XtpzIZNJ1C-Pd6eI3d%jG9;ou;fp|#ZUx8bc8LEfR&b8AU=GQK3int2YJBgYv}~f)@@Mn=;pC`P zgC+`Z;sTkj4h}gC*~W5_cV4IT@phPpBT3}xQCYixZ&9KZ$&I@*7GxIc4FPg#c=$mW zt~xviUs^nI4dr*SG8%0gEa#KP_Xq-wRC}Pg*iZH_JMf1EpVPfDNA{5zXhyxyrXw%v zX735H8fjqO?keKxWXm_xb_#k!$xGzxL1KusG^5yhy&MOr!>Y7{oaLtZ(VsP5`s$Y1 zb8!x5{Dav(MZ!6$95a&7I`1c>K}!FfT?G1`n&UbnL>hVVQGu06ded2LN5&Tqet%Zy zD9b0(;JWw>&(aylKX8i3o|<$_3%Ol$S5RW}s(2}dvi=M2YXA#985q<}B(+xu?Yoo* z*+3n>zTk2K^1y+|g+IWpbk@dO%zzA+UV&;Zu%0mQK5g~s;%58cJ zI8E~6{MQ8yiM|Z>ZzeZ$S;L$6*4HB;N0XWZsQsvl%7Kj%k(->MkLRN6=zU)Z4pxYa zZ!bWL;=A0#H=ZWpxA`CcK-Rw(f(x8PwFa{PuD@knl}7Wy(F(%acAZFmIcaD|$7D2< zHVQeP><`W=WfxlH>?#^RZ0j&v4vQ<&GZ{S*)Z_m;Y3{^BAP7*L#DdP;-#GQ4Q@#0T zPf>QMBX+E)cQ^(Ps{*RqJX7pTj{A2OE|om>cD|}<=~|gEB>ruWd!K0@%>=hKCu{qy z?KhtQjVYY_W&r!1+`y(j-IsqF1Myc;b@HxjWksO4g)CWnj4rtm49&v+{dM8qQJ1##rP z9@9n{$Ns=Yb0uzUwmoeoYazx_6+p#Nb+n})+O@*O8RKQIsX&#k)IeK;6iYv&We8Wj z*V`siV=CV-kDxGu9=!MB!7BS&s(YcJQ`u0F<@~E&ZPeuRBdA6C8C937I&vD(vGb3a zEY($%V4oR{y5UvQQ69hG9VTi~d22>eY2M#VTT*sddo%{_y-t+jxg-i?C`W$>X5@-O zArb^Uj1nwQ!iWy5WroJ_x1ebqYU^hL&f2J#dZJwfHZn8(ulF^0?m=RuZ$Q5=;g-rz zZg@v{m}Y%m_f=V~>l^^}Z*Wa}ioE{u7sNw~OKzUry=&8b2<5nS@keBG}Er9|hrhANrbFrCwJO#aE;lL~y9bQZj zv}2x~I4n1vH4`qvncW`{$#x)MIPnB<>!fd3@ZL}IO!2C z@9g5K9O&Z$haARW3w=rPGA*gU{D?a>Lia>86K1i!mP-J z-GN~hN;B#qc<1v{Z;M}e6jjhFhiKP!7jV2Vp9KMU@t*K1SIF{%g1HQi)&&tdhZOyW#M_m@ZFOW`YHBylSs;R(L_;4Jt*CdVN2C2Uy*wm; z9v^%iyuZnl;ECb=m`>DbvL83@(vf}z&b+sIEx>SVsa0PyaX~P`MA!4qR<<)g3DcS! zgA&Z?H-aMYHyD)1aB4hK^ksP4X;X+(nZ$5c;$C3|Y@UaocgL|$0?mofxIZMk?Nqt> zaP#tQk6HJv8ffN6I$)w7*pH^*$7g(5%^0u7O*qs#*1R^e?M~nwuNQ-S$-oSLNB?gP z42;pvuB}_(eHMfTALt&Ydg`~+mo^C*y-HgJfXmPBdZvYN8}CKz);BJ4{flqI67WV< zglQQh>i-v>bw~De~x8|6jr?9oru`bI+_D> zKX=HBGwt;-3FB}7C(1SZdOnmX%3c%I5}bbd{k@MxQsvW{Ncoe{PO?(7vvH$0=;53g2a*{2-ncs zAoc&`z8yxm3NJGseG}NCRxNsR&m^Vw@=Zf0^tRK(P88El-szR|lQYW?6h%#JNPODa zWqi^?S3T`qj8WUeNlo`Ejhv@y6z1c@jP$mt74baL(=xYUMJq=Cjfd5@N4~bMaVhEj8p9Z zm6I#pDrUbyS8~rUlNQtK`(jj5J+f~-#UfPb^Ww$lY|)Z?#o`MF!Q^}|?sepZbG;x1 zjpGO-ELoI9!@Y%$<<^NW+Qje1KUex(PM8v9y;_GJ-)@@WOIm0Q^SKrk;N)l}`aRwB z9Xm+KVkwD|`jpKTOHT~Fv-Y);zSvC&`Fidy3QxVP&ysTyyc&Ht=Q@~PIqVNT`@D~4 z#ILp9!9kOAi&&Ps-KWxkCjrs}kn7Ra21cTJc$swQK9a_pMK{%?Jn`r3 z^ihB3^@)=Uq1_L=7QIlOpa+u9;oFJ@Y-ryvJ5eh@am2KSUX)V2BQ)GB{?@Ch(JtwX zN<=jE&@cNh_&~Xr8&;Z@@5dj?lRmxrhT98b+pkMDK8y|9B%ebtsN+n{8sbzNYbqL3 zH!o{BQQphZAWqDjTZB^ku{0s$CaPoQ?HZvt=zA;1RK0SOUr$P2u`#`W{c5`G5+_|W zUH`J+CtTD))JVNB=EEjsYntK5-;ZI}>|K6Or$5NfSJ!1p8081*ncJ(1VAU1mPa2zg z`#qxXooC9abXU@O+4h9i_l)mbL{@2a1g?R{;PyT-IG__w4DdM~f>wnq6TG#7xaHy^ z{p=jMP2nqS$8kNy0WnV3(`RnHyAf}%d}=8G9aAU@4-4Y^(8eOtXP995kgHwO_Ilw zs*nq>00f1l{l9qKXkp<0!SujyHTPK zl%V>-_$l|rtknEQomj@hhlI9)^=k;ayB#W-moPj&a)uaqlkCfy={VI)+QRFfYhY1v z`FX86^jw$kd%#LH7ms*EKDxcy`vq~CSeLb#-bnGRO@4cruvkymTYME0l1z3LHM(T` zYH7p@m;3gZ(hTYNA$zC3K2@YHi|cF`k?T&h{1=3cz%d347b{ZA)!7}Y)+~Okv)av` zfLg&FlYj>#dVrO7?eBfl207jERxd{$RUupaV@^6m0TXU{^V9sr!Bfp#@6Q8QuMEXc zPK%OVuqL+WzkM+&n)YZ3ml-%NJ?T#5XIR;)Q=x$4>KV~#5j!7kh~5yqmBJR z3~dHd!nH!r_Ts_4Sf#@&4-|0bSddUlN6ur_gDZ9D%uF>sffht5nz8QI*X!LY(xTnX z=iw9*=t+CYJCzW+3Fr$5;DZ(%SCC`N&Bq7t7e3jV4DI!N5QLR}P<4DOR^DX}BLdO1 zKEu^){OoJJv#-4cCy3lWz~^A!7j%r`aD}?~xB>^02X%QSlWWYlfUhz|twP$&fAour zeyUt(TBN_EK&tqJHWehxWbLvpG5h(~`uEAb%rynNFV=NM7q!2AqZj)o7jHf%!id0w zDn?^Z;SOKq&W*+M@Ot;|i=SGeoyv%;CbuLlB^V!JES+|=62@xJKJzKpZ050K@Hlgo z3J-4~FDf|G`U2+@P=%kIM^^^qzk_RF+HWOJ^=1XYc(fuZW?UW zI8jo)?LWFmbzqi=3I)e4L-!CF|+Pe`8Y&GMY)= z*>2I9kTb=>V=Fcp+I$Vust&goRr!KtM%fUdda?!{?{P zJB*z$#y6j#jXW&p#9v=pk0$QsPxPg-4hmpU^|x{YQL5e9((x0lbw`PljsAA7noS?K z*xMl2Su*IG#zyQ?Ax*>lrif9MCQpcNW#pDb5=XMNQWK}OEoqR1AV){7`!PDCa$2hY zMos;<^cn6>Vz-VT;(@tU4khHf~#D(BUU7BG0h! zL9W0l54%-#Ep$Sf(w+yR%suxiWS?nmwro3CUM+*UV$)<@qrT$XV4@K?;nJ?`wyk7& z5q9+&o%S=%L(K+n}Gkka)2~NFQ)sKWsuzkQw06F`u#9 z5fXK*+4bsDNn?VUYIkj)a(g`w{`x|hm4{X4N01NyCd-@)J*qvx;h2m4c&V45EKfgq zXQ045%V&ayMqS0iF9%Q7?wgY3V&gPF{YYEx^+UeG&%ptYgCK{J$B(aPQl#Cbb;=oU zzA4|8dG|Tz4k@)Mitj8dP*4ZksXmA-7qYAy_TYCV|K9yZBJSBvvxen^eaHpbL>*Q| zN+4}Fz#xw=;$48y0K^~65YfJL`!DEW8;sgtC1=$&!o}Ii`0Aapsc|wePiS|^g%tmu z*T#j`k>hQ=`dBu6pz#pTkHO$Ln!c^yGNIJ!`r~N&uKy3!*RNXqG8A4Hh!$UpyC%)_ zlXW4=XhSMbXt`z`Wn)u4Y@RYDO{Uhwd=}d1QqpZ=Xi0bMRTRThwlgCc>NS>ChWOuX z5&XJr&NC}aqMD72u21u#2PsMufj-P$XT0H<&`UKmAR?{zfHFC3yhWd}wk03V0< z06gN946tZ%M*Xxn-zk3@M7YZR)TKC=vBTJFyZ%Mage8JoG^WZMIP-pP6ogkIVt}&Y zKkup_^!^25-EQRT{`%O_H=wfW%d^imiT&bjxTI*OXO^t6mo(=_-;Vz_kolL9%>TX; z|M&~yAaZmikPqNME@w9)8zHEn^*TEd;b)a~Z)7%e)nXrK>;70%qlowB4v9AU2zv#i zjywGU+)D0)1lAnlO}n83NHQ`rAdUdZ@bA*maf3d*TyrA~Yms$G) zcv116n%tgjP2KyF{eVSdYV%jcXUxZi{c?nbJ5M{`^wHqf5!Xh~@RTh_ANP~FsCwt0 z z{D(B_pR*$+57JRs+j#=2U}}wpz+RQOHUBfQw&yCT)zGFP#CFUu>~$lXw_FQ*k^_{h zzj$NYm@4pvS3+=lrn6P_i7uJdKXE)VVVo&J(<>UeK@Cou&rr0aX>nk|=!fZ3TD+@= zn{ZnrOC|li_Q{@6A4!-?6DYeAS|W@lel zeVN(a@=KC%2sy`5dI_ zb8RXIKP|wg`F`uHbTlGSUT3i$cT4ekl3#9WpGC(P^xesZ6S^ZIC!Qelph!!3Z}ZjU zgVVhM%0jmhnJ+Zxgf9;UpI7u4R1G>ksjRlR`yx~IG}IK#+`L|5v|todF-zoa8DM4b zJoM;FsLn5mFi;Vc`&i8*sf2qQyqvOEpF+fE>g}0l=V8*8ZgKt#^?>uk45e*-xtWTs zYh@wu)!iYgwYDoBGl|SR?{@#f(f(j2!`DBQPnAE>z#=AEVj3w?e+Se zEhvzRZ68Y|<*lT-OR=kdWcGBW;uXy_fS!6CuGtZHSPc?n*L#-C)_lprHkTJCI~uL~ zY3~Oq!?8S%2{-Xg-jUl{rTwTad1bx| z-+Omv9i+Kge_B6zLl#Wq8hg~~3ps62xEMfw!iem?Hv-=P#U`M)M}f>F>;MovS-!gj zJcy#PMy02hh6$V&;|?$UC97h)Uq>G5%9j0{dir>aH<|NQFA;yP4`lzI$xa-jzdx2g z%%kqp&7I>VzYm_{JSI)nO4FmBzOj4f#JCSuI-+vY=~~CZ5=UW36^|QjO7wK9tAdUL zc)wsIJ_;w@3a1Ut$1!2D4`sx_Q%+=pwLNk-!(||{O85!vRzOjBI0RBFD!`mTqz}+w zshPfQvSs6iAt-t;au_MJO00n+8) zG_iLSxaB)Q*((#b{(z9xeIByrA5*305hnq2W^JqpL5u7)dgBUgwVy;zvXq%8=*)=WD z$bM#R^RQPun_eOA{znhsZ&{4Ufsy|OSxoVkCNzy8kFN+`n4NQJjq(f6jcXa1#5`GV zv^TTnj1E5UtU+lQP-H!VWIiG4*S#<*jT0eg_C=yMm;giZ*`)T~b>f|Gpn&T^?3l|- z@6~_>WAC``hz6`X&CRXc?;l!58m&vLof3aUCBA&|b&hBQa-mwD<3C|B>S9cL%caG) zx9XV47L7?uyeeuaW&lPN2kHlAw?aT#iWlPmuh|gCLaVb=t)O{ zx(hAP{wkpNtc=Ez|A1X6i403&H-8PI7sk}9zip_XC>kUVrligmwdnlF5c@rowr}DX z96yVlJYX@+o&-98`DdUN`j`a;-;RnS$RM=sLRtyj*nkS(`<`Bb(3&bbRr>2<5|1a# zU6ddgu6*KMfyVf=aP*=pj^)f-gb>ZNHH-Zm$i54x>IxX14lGg@b7aWrl}vBh4pTVQ zYW$e8;87xpQ2Qk7pLdCY?-zok^FcH{7d!vYl?#;Ze@<5oLTv5HR_uuEa#IaXeqJOR z+Ixaz2W0oTK8OCI#P>|Hb9&sU9aa^0n(PE8G}D3Utc9C^@Qg)}6gToYq?i<$)C@{q zy28BC$C8$)Mk&hL)qF8dW_ut=k&bC^G!DMl>kn=|0=)6zqn;Cfo$t=L9cV3)GZYA+ z^NA?fvIPj%*mT-UL{B?Gt@Wy7Qy<%5h22-|sU2UmXLTJck zI&ObWXE;mY8I)JG3Mj-!TZ~V3@!4Ie6uXts#Zk>kg#rGRgN`5FMpELbT#tZArU_oj zf0~M5Ku%T@65A~v0zV^tWVZ)S`m7GQDrE3S0%F(~Qn`96wh(j6q+-eO#>f)|CHc6v zXQco~9TK5-Gz`4jUsUNNrOcNzb>|WHGRkU8)-c&0ej?-l41{s!RdyMnMG*9U0y_ezrjNyj*EXG z7yFhd6)T6ih-$-EUNY5@b%>{(DMXgB6`R(R9 zl;%hL*{N?!COdqi+_m9mp4!rPVeC{M`_|-IhI}>gPy3}b3#vp>$VZwG2t*;DxUeQz z@Ez})1gDf;~w4zN0JYV3&H%0;ft# zlzx#2r?+KKhXv!1JGEYpmevr(FNUXY-%u4PMrRVhj!Zz@A+!MzxxQq^2??4I8(EE< zPjm*KiRU#NC`dra!z8aEi4`d(PsQqoB+2)shAIW~PeT)|b`o!w6{$^qO#C6!rzAGZ z4WsjJC8%w>5zu+(o>Yd4Ffj1^XdZP!K8aizM&+K`oCvoU;?quaf1lnJa=Mv)%LnW$ zW474Gq%-2THub$>2Z*RUxCa?u*BY;$_%pvOS|>5=pzqCQ>T#C8a4XtPiIZ&eJg3*W z2R@h(;#kr5h|stLM(*fOE?8teEolAv>6U&~xaqBW3kS{1LQPIx%~YKq_u%)zjB0V4 z&NCXrSv&-Z{{!(e+5mTlSEprmzeVSabAQz!YOasQ7_ZfG>=~N-o_*j4T$o+0N17M- z?5}kFeW}OnGy1k^*H>N~I$(VZV5oK@f^Cug>EOc!N4L7H-I!Vc)UU2wy z0?Vq56DwpF**yBp57dM!%)`6Ffx-UI@S0XQmeyLE_{IR)Z7p7tDW+XUo{LmPHEdm~ z&0XAaF9kb5Kuxj*Io%H+LOvgW!I?>OcmTwTiP|98JvBZTjcs9aT+!t9l(yL2=6<`B zhYdY0x27IU`6PZL3g-&@0f7#W9u+AurYGEBd zVBYVQz8P2v6y=2Eiq1x@o%57cr?< zn=&_v_~l5m+vds}{WVe+Lbj&4rdeXznpS)eQDW?1qCHfVIx-LzY1d1Uo)ON&9nwq^ z(u^{}_h6$s@)j)6UQe^89i+StU(U2Dq_MGB$NpqI76pT5_qfyDTG1Cz1AsN2q04;F zI&X>+`_k4ePCc5O_YDx(!+z_2vK0f66+m51RM)B}_?uDm2zzP$2@3%g_Fr_Z*L^T* z%^Ua1sbbZPz4Z~|ooQ8Kl z<-=ju=YaEo)zPVl>|UgZyrIUDL)3joa%Tz7)sDjh1JL zfn4~cR@sBnV($>0qzA!TE~TAJO@ch79xt1bwHGjod9iOAKV@EMlsc=F)LOLa=JIUq4f+mdBcat z!w1HPtX0|HCsQaR+KB;e_|(H99vPufj>h4eyfyVHlgnu;DgL*v^B&#?f6J>tdhfG$ zMDxOC)|d}31-Hd?<;2Lse)M@}^=%)-%!zyO6SKsjGd^{E@LEluey z;uR%H1yvuq^a1a;b6wGyXKEk=KQtUF*$+@#_zR+6lel&OFpytYdgP!>b)es|$DnaA zPz3N*90lC5JFYG-^1V~U$pu=!&lv+^pJHo-7-4cEXTK_W-dt*C zNn^tWx}+8UXx>KbNKV=p*t1XL^tnZ5<(@gy|#ESP;poyrzYNhV0HWD!O;;_6t4pZS1fC+Ea5_Zdm|nG>~YAv z0wN?N$mihOVt=%`TuZMS*Bb|qIdix+|1s>YmE{M$Z>2kMsy2Y$9q zQFQ-;6j2^bivF1H@Fr@3%U0$4?w*qgp~b0AdeThZ)+JY& zQGYHvKK66!SnY1up338lvMHQ47T(&;Q7EA{&kH#l*pE`S&*MRn`(u%tP9#E8~S)$AHSUbSU_<>@k7|M#UbJ{VI)vC zu5CstN_}vN-v0Q`TfPSdxp4oz%^=x?c8IqDpD-Up7+`#VYyu1ZM=e3tiKuEU^q-Ob zU6JG=a`4>urMd9;+d7gt$yy0ho4PWqy|N!q=Mlz%BS;!~?F9sry~%Re-l_-3Q|l@n zuJ6LU&M2Nm)S-sI?T}mStvLWO6v!h%pKc1M;)B2h5odqWVUfy=*MT53I9u1&Cz8@i zgb7Vr+psD0LY51*&G%i=Wc6}>^YAyd%n@IY5YCmaBVIez7`z`ZO~+5i-H#P4`v=6= z$az*$X?lW+Jd_P+A41vKkRWV>G7=SVg<`}*>3Jv}WrXW!7ZQxCjUXkWwV zmo>Z*&Hb%a@jRR{jH}k#6aER}Kk6c1HA>g#BG@i}pNlspvP@KlgszMG5LJYQw|Xzo*n zrpSTR<8fqR^%7V4suFY>tP-;2!%eZT2nQHz<%)=Nb{dBaGl#p?PTL2@d39$^$)#GS zvys$%(^>mbdQ0RZ{vP@M<1{NXZtT(aUdRZcWEMRe}zlJYDxq#H%+;Y5E9EtkLx-OgT2a z|NZp3DVNgCPOe)aV^)9eXSn=m>^aMCCa6)9Ilq6;f3FU^rnXC>Bu={k z>Ff(sUj67l4v?5RwbTibNcXxkfi+Xp?Mi!66d`dY*jgk$56w1ii zICm{RnUQrOsT`o9c=Mu_W7#jW8aycyZZRjkMgp`2l&`jxV4tGfN_jsn^MZlUf9D=j zs0FSYB{ohnP6P8#_yx+CDIWXM-@d&L8qT}%T9_@jd10&bR3(7=g!fbd2MgWaFE(wH zqRne}rI=Ku!aOfhAr-DZY1|{)>5TF)opx-iP09Ld*6raVv1MUQqqz(LGq@q{5X)); zg+uCuPP+@G;wQIr#|Wu|^5^ovG2&~j=(%IWrA1;Bf&l&&<`=xpB(x$B+R%9SedLsh znsuZqKX<_~^fdNOwc=%eZd|#)CiZNY5ayLTK4#JCCvlkYZqQm^Fq!jZCoA#nH6fF%e-LysF{X_BmB!_~I{=8VX(TREv0=vor{ETD+ z1Uw2jwq*HW?hRrSP1*Mcopj)n^mD>m@e7$0i4AJcHo!MIk4K_n03KN z4YH^<;L~#X>{fKuX5(0&?@>SDH_+q6A{T6*-CDH)LyERA^>ByCH3_=-y5xIIoKOw!Gv9+_ zibk2Uh%0aL7{o`lC8oh(i4PC@J05W}4Du3pmZD1YA*fGqdt@|_8@%#Y&07g%yKLHZ z(6Z(Do`;^9syFLG!mF6q%Z=*y-S*7=jZW-U1WvUFuh~uAIo7}0^=R`M{SOVY8je)B z-ntSi5<36`I2I46mV)%aG7q3$*p5n_KsGtSJF28ab93@vhw?o8p~Ay_5v$JqoR@g* z)<-vNaeyl3L@@a*PX1=@#FO9Jj3pGDpHq^k0?2;K96~FInt-lA!(SaJ1ClccMCt@4 z&{y4!Q!&Cv<*e&^hhA~q6kCAXN8BMdG{;|5hm^E-ttYJZwe0V^Y)DcSvP3U~54-s+ z>dM2)_x=k?3gkqoJX5})&1w%kZ2vkiZPFD7e|;Jt5_T%?RD7i@f~u^W{>R|0&)?@k z<#ro0J4Op^U~?3&w7W4T&W!zHzx^ztsGkh7ll>|o{i7Se+Z-?PMPZ(m?5M3IwmRd= zZxD@k>m{s-Upfx4fGffB6`6ti zUGU!v`g~zCDI^aaNWYji{)wHs(~L==VSDV(Jp@jVDHu_pj6FfLJWL%Y{8P$;Ct83< zqnZA&>d_u;dg0JAk342taHsTf;e|XBFVs_@X(GdjP^fSf4|NxfWwx~YYI;C}AE47g zKbL^uascr`KK9qZ*LB$kUo7XZ*P~=e0US=qy*g7{=x1z>;#d!{)04zf>Y?mjhVVa;E%VCXMt# zG=3QxoNB?0eLcA!Oy!kYcrQ9YEI#7)Hx$jL_=k1Zy?)I@2VeXh8bd5XZ1 zu>JI62q?K#ZPykONYq&=O?DVIA75CScwDN_j=#_{OyN1Bl1=w&n@ z($Du3dWHXA`^k$SR$i+!Hrkr+v`m%QDHy_bQMcy`RRGaNpF?Qn&L2oX_lcyx!7u@s zB2PiD#IiID%GX?MyT|a$Xf{-Ul!!z2u&-K?t%xu%BUdwg^Vy(nd1;x~H4c|@-gFZ% zFs)lLw+=p*NbZRrKnm#aeW_ASvY&YDlN}WBLLIu$h)BP|ovvXUgGrIPzB2d5knDVT z9po4>Hwuny7DvQoi3XlFVW$9_d<|i2e33Q-G?pUPb;WT#?Q{)KPF3#z6r2A>I=*;K z?xOySFNr$e_EEo00K5jb%LtbZYwf$J?DIv112JP3_4rS`^W#AM_9_q%TN2`Y_hpaP z4~8R+@IGI`?2QKGpCO7uB zQm(yJYRaTx-#$phhYABzd><>42Y(WoD%9wLy>&IZJMX7E{zltw#|HA_jM@ZUXuac2 z&07N#ee^r&+xo@T25kSpM6YNxZ-a)s)6dfyD3i0~rzwb7cFJ~!+B>NtJOYVy{t+zv z{WfoHhhEJDDez#BooFUJa)J3v7XHP=_=Nr9-42Zu!{o>})4MdX7^b#WWJPtH(>{lf z%a%~dCS7w1ZP{m^s3~V+o^`X?O*X1;r*=5_2aep6Q1_{9XZ@&P1@#7>Hof{XURNIZ z*gIeIyDb4%L%e^p1+(dCczDG#;)}7=^6jo(0pBlKH_VxvA75^n5mjd8-emF(l9>$; zyy`;k&2K!aPxSIYYRMqtgPm2s^|AG6N8E|Vsp`#k4@y3LP0EAYy<~a!jyo}J;#2-G z+@lw2J6+GaFGW87cv%M+S<8a*OlQ0r1Ez$yEqs`!Z0m7(G5gFh-}!2^+CzKYNBOKW zQ#5{-jj`?i+9zzSf~N*`@2xx|B)V0;KdjXB#*%M-gmsNhK7u6AAuIaY8{C?t(+rd6 zE8cK?$dCD%s-KxeL2mv^qf_tWnc%D=<}b*k1DkqOy@E{4X_93Z?}TRxJwxr5|x^P?c?ce)rmT}o0q)ktcduq>?3#Pxx6%qw*?+52)%Awcev$k z%!$Ozvu!!&)(nx&oI}h>UEuS*P6?vr&Vx97OyBAKjlQ!Qa58OWl*8ITxpV@r&-V+G z!KNQHM&@r?DnlQAs=N0p&VRIRf3JE&g(?G^gimx4K25M?X8~El>_^*CKxM6;H^Nf< z_f=5nAeE-}-Xq`PmJqXNX+``;aV}qpkiK3p!7e~FU)8uqL8eJat^V$H! zC10e-F*XCV&S%orfFd5U3y+1$V7*>fL}?+}?F)*82e%@dCbQc%*YL=B;43WJinEYf0l4F7bq!4-|P^w{luZ^;{Pyv|=NA)r{(ezd`Tl)!zVU zup>aeD~}OMoSAr+xgcdIEp~+`UhM-wjTy{^r;Aup5+|lxIS&T1Vl}dXNi~=>idE+N z>TsPn0i;WgbI43| zjy;x>*3?T`wD%l;9SnH>8+}~f%dvvLfHV^LCM=)51TptFxrMqCtRRKO^YFm%Fy$fd zK{Z{`?|WeaE3E>pW5LuqW_c}fM|?BXfHIKGAcWsr+f}+eYd$$iDP86Y?KMPcW@jcqL1SOQoQr zX%w}^j~iw@W)BjMT$k&dUb^Cq8r0UW3T9CiiC@|h1l@Fhi;^aQ@dM#bknn$6qW@0C z{#)Vw|1S;uSB>%aE0{xn@x#e_!yTj|Va@M-)mNV+4fCq}CN_m2h=IdD(vknLQU7A8 zvD-|61L>FhB~KcJog@{Mi!*wBHVx#Ta%KJk$wZLh>16h@EYJ$$hj&Q;acsQn9}8`iDnN+;mV(4 zznlbE7b8`vO0u5>(~{6l)FAPOjidm!lxv;E-wXrGwxo_x_aZfq=AeY&702wyCiVx8 zDVHbjmU8nadgfnfe1X&HaCuF;z_IHwm+$euEeALToXAd}?%bYY>#~%yJ~kROg5x$t zPQs3i{=CYht!$+8H^5sJdei74%hC1WK@*) zcA4KQXHG^^Z|3WL>S^1P`&wZp4|CrtT~TqC|htv45u|We+st@FHJqwXif5`C6RV$uXknM4 zdz>zoe8?H^7+{PPW7Olm_->AhXlNwV4GDBciy!fYgk~eeF<(CMa;JR*ap5+GJ)tBUmfSFr~<@=uc$yBT2)l!;1W zNr_+h$x3jd2st`-8XDwn3D&kbFAI-tLiFim&`)UCI3;M&?STn)W|4u;IC=2&9*4Q zZg%p1lI2WVKXfem93;L6qqQTzA|LMw3$$v-8d9y>J13AS=Gn0o-vcYYe7}L=-e|Gm zq}kx}N)DSB>r7LW0&iA7it@;(^?weMf#VHl$o^hT8F=Bp7t^QWEJrJ6%ebz|@}3E) z%?`0R9fy=i$YNZp3vqfQ4R)JA`Se_h?{)XLdjrLAafP{5&x0!A`gZ#ZZJ`Y>GO0#F zLBov2)wA4F*q3!LUw_Q_H$gCFm0Yqn>ag?_5_cMP8p81>SAVu}nC0kYf zTR-woj~D%sy*h0djru{!>-Fn`-_zKVc%Ah{)l|((0d5^%oTYLuiwyfpiuWcYN2l&lV%X?$mZYG8jp&$;>qf;Bi>>MDTsU> zCZjcj3P1axtOj>2KTV=4u1sz@LD$JS-zy&YI<$Lto?b_UA@&{?ff3+mXpsNwoK-v9 zvUk35@?TZ_`6NY!8S|}bPTDHhb*9l0_!Jb1gh4rU-oUV-+1fWlC2l{X=jv?>+g0)1 zA_{Y9X+(lpPK{kAM0#WpNKBD952W_ccIv_P#_ufES8bj;^X@*Fr>nnvo%YZ(@%w79 zs(;Z>ZAbtPD>9!Nddfl#+LFU0Uyt6rY1-vCl)$A2hRVgfj$S4?T*YIdvUmpU0g`dn zOb2sY;)X3!vWPeRvXyW*T#mGPy|4HsqLgoF9TVW&ugzT*Aot6znk%#|pO|itTa4`x z4EPbRN#AX+{dwrtE%D3ehCQ$4&smu@9TUhgYCf--7P2}cRyrh9k03?6J)5qPi1oj_r##p%~O^s=1PhKoqfh`;eSSXG*{px4}!<)z82cZhmbP4xVH+JS_-VyKq& zqhp>qmNCvL;j6IqLW0v*(PKB!>p3rZg3Pq9ogjEHt^IZi%gF$|sp0 zZX9~JI^m*P;m^`E>L;@}HZ1FgLXvN<(E>6JjziD$hxqHECf7}t!?UoPdn;C6DRx8V z&nbwA#$J8_^3`Yce?h=V0qmB!+K3nqI>xZOs z5N+wbr}k8BtY*%(QNaN@_Ib(2pSrkJXh2?c_7(jt`pb+YL2}l+?Ey?mPzCIlIjJ!0 zVVg*?I?|&6A@oen^ZlJmUKdz{fb+M^%B^*SKwd(hENzUxj%x|E+O~F04Wu{WYHin< zv|B%hY13VotT#<(qi}v*TW!K``sMM#j5O83CVgf+?L05%D}<7T`Pj-a@>77Di;c}C z3Y&CgcJ3ANpROwc)BZIlu3$89BTbcYwyC|vmU_6@rH#**vtwV!{0fbRtCMy6x?h8i;fl*Ch=34*ot7cLHw>m+=#Pog+mc6KO-$RsXFp(Q9Ve|jdt9xRCtlv*-b+vp{YK_FZ~Md(Y@eYOjquKH4x_3ujVNDO#5re|dwEhhEI;#N+SFH(Cc8@BABj`WFQC|Mf~PJfFDWJ@<-T zGy6$+q_}ZJ6U283{YY>gKJM1U0u{iW<(A4@+sg_SzTA3A{4-B1Qvs`FwIS*|I{o(C z1}0;Z-^0}wtJStMp(8~9Za%vH`J+uRQiqKzGV#(DIU~_9j1#ZH31hSa4D+|w@KXba zRoSXfMW&S<2}pb7g_@Y`%J(+9O~e(+ytX8qevzl&|DN~#ezf!W&(~idhKl~czL8@a zdE8RUMNagNnD84s?E5&Ejn4ULuQuu0PbI~Ed3<1WNp-BGhoO8a$!@Su9$xyu7JC?0 zuW3_gwpLvmU*RgCd{xN7jI)66fw%lZ!`43pn$`oIB;wlg5uADjUS8Y+`5;jC8Q-8uIoI{>pWkJ6X8w{XjASB=#Pl`I7@jI%uHEXG~d&8yV~0PD&+t>7=NyVviypEL}cxmw(HFZaCetlF$eW&m&iZ{~7?Bsf^>f##~ zBqNAsWmD@)D=%ztCwo5*pY7;!#wOK47J8P|$rJBdO#$73v0Af>L3ZeH#}9}{wP7}z zHYQ|oOpwhKoyGcm`knT9;r4pT%|*To;#;-1io-89K509>qs~PusA002p5!wyHerjO}=ek$nvDMqSo$3$IXpb-MPmOT!X&3FbeG z1YqVPN~pqS5)BWoQBr(YN;TXcJoKBgh$b1aj=di=H(eyNWfyi*3zi z$1@+zzQl?&Y9L**1ss0GN0#BfnLig!w`{Sm*>1;DKJ^t_s=Sz@(<{{Ec5v@&UBqHY zB=f~SZZj$`?w$m-E#`iY$=wv4mNV&%#RvKIx#RP>%?HCeG!^03e^eNm6lgqA*ih$Bm>;Bk0%T4w z&w*~w-oF!_X>;0VbEmPc72T%lrrK%a?=_iEEZd8|<=7`%XHcY$r0%0W0ue<~fDK%L z;(1Vck`v8cDgF-zzE9yB-0sZh2LC1x>_U=}c;PQmOBr1?M_fIm$K@$l?)EzJ#5WO- zRs>Yz9f{%u!Fh4XF4KTF(Xo&4bTcoH@5m)%F8o|mzkmL;L^>FdUbPljuQi@^Hv7Yt zs^j4W-5SyrWwp2IhNkw)4ib49a>uc?&w0lj`F%%AZJx%*q$FM~z9LXz1l$~f$*uNj z!|FemVo{pztc!%~)~vCY%_uv-U^}I@hRx}Gjx68|!bfqR% zKY^X|eE#nH%wLjmIqw(?hBWS{KQAmMHtw#||Snq*O0@*ygny+n+5wY?6C^71C!WOpIfp!bf=~Js*@@^4?jl* zU4e-7@Y8!)t0LRB2%tH4(`4o37<_-3&rl>5!n789tbmzUbaP!CExarLT3zZxu4g}E z@|0**`O*9yDjT_w2NOmPf{S440h;L998*pxE#;edx7f9#;c_%WvvkI6$$oc??^>dD zwy%;M&~1lvH4E?^W&yzdG?EBPa`Iz|AR)YP*nY5?1^$kdd9MneZOq{9lzOhT{>Y>> z^5qr|j8%^PHx;*I6~c*%$)}p+0*%i<`=oVQM*Ivr*rkNeTDDTyQ`WwfT?PG|LU0;@ zN#p&|J!__!lSU1FE)dL)Lo$cLoxC zG$aMYDlz!o1E;U;Yo>k~cP(@%euvs<(bTa(mh5GJqpP(#;TI7GodM4x7aH0NI}bD( zQyfO`FNtk6ggjCsxBulpzB;*^m`I{RUwi)}XDYhoSNkYvF1T(C1B3lq_aKG>8V#3-E-A)P8&)uewHxpdU zuxNt)+{?K=u|y&RSfItUJ@`f+uE7!~7Y&qD?p&^J9Br%{8|AQbFoj<`_tfZK>Mw7~ zP)bDw-tD`V-yT(GnQ+AjT;r|d$Y*$7oj5U!70)g7a&DKd|k_pevDy{Wv|WZN=JU769oBgU374!1v6=Wqx$nlMCQ z*nc&ffP3izo{tojjXvII18rP2K67AO$58DCL0n(s3F_uKIq9C{oH*~sf0mlL9D1<} zF&~h%{|%3si0IDhP6%T27k*)Pe_Ko6nhbA0`_Pc!T_45MY@069ClHaG zV`s<9Q2u@%HL=*egDBQh>dNHJc+A}=R%>K=Q$(#$fjYrFHI9CIMV@*=jB1*4xY#*`0ciuXmGK7G9D!^O*`$3V*H^u?w z2TVJ_HdiJ^CK8J*PP4`F5kQ?$k~ns;ekU}1hlZzn{}r+w7NieKBHO=^tE{KNo&HZ{(fIhD;W>jyqVh+?1FdM?!IH(QFE4_MnhffNQ-Z%c zp8^wlh!1cdyW>eb>oDFgbup8_c4T%wykZJV(QxPD4!yl7pa2v;V@?H7PZ!|dZ{(k2 z(A0hKkf4LLrf?1@tH-{!sqioT{z@+VUfNGqt?R6sS1G?iIH0>ZkDeXLfl2`tse;50 zcYi^kP4wXRgXM;}qN9`@1a5r35VQXTni`X_fa;dyLAH6lYyIpjSu|m3spTr(h|f3T z314D?giB`ul2puL>cJ<%M=X*Gk*zz}DDQug$2BRwQTeV|dpY+ORS&<&M4o4GaXWee zaf(}c`+1vm0-6gQgOn3|yhOI(MNG7OfMzn(IVeT2Uq|?iQRbmE5xNqNz*t)xvXMCc z{A-!urwi=7$@iQD{R*6nniwoc1Y(pM>U3{c*hwruHPQ^Qf9A1m=?Xy@Y}DhrEpca~ zs{ODI*y@H63+Mw4pXB%Ee~A6xZttS=g~PVPwyLTO&zSYWtD;*IS%KhWJeGlGvh$O= zA2l@T<@?BaJHy89N9-3s)V(m6_i24;d(s%Y5)1h0XZIu-oG9;9S4Of|X_pLD7P-{S z2Y5paDYGAd!iniuU5ug%oclbmM@}Klxjf2iyixB$-*_gN3{|gk=>s1F!9e3j)?;N* z;ZAbg*xXgAqG{kz7?6$|Zyh`3ItbkKU@q(!zfSRJidW%V4ZXYOY*ce1;Rhc6;0KHv z;b#5gxo=iN15dpp^Rr`kCc|rA`yBw>YBY6wxrg4w&^b+NXZOe-q*@#{__@qR`q_f5 z;*o592Bg+)Li~S~K&THea-!k5>~Y+gRE z64Oy6DHJ7sPWknsXR(am*n%Bo{(6-APgPAdSiadu(!d-Au((IRsF}TlS1k#6Qj?Ry zz9_9FJuQ>mh~Q$6gh)a29+CBmF0;+?z%Dc zA(kXDMiR7efboy$7upwf^Y{0B9<1_c40~0?| zj`ci7nPAqH_8ZEvGvdqVzQ3`R7HhV;-N2ZSN>@qHr$7*TMQ~@hr`U%Kr3r2@)Lj#)Mj+p392X@sIj=K)1u1#NUiU;*vsKTsQ>c zQsY9Y4!>DX>|$+tzBO~vleIi)u4(ogK{|eOcEj_woWD_en2>F zZ}2%hpV4)Xp{ZdnvQme5WI>MP>hj`9OkaVWda$+n@UH}}-ceow{=@spqiv~E%D_+o z6@y;zBc2}P0&t8wPsOz+ttp6R{)+}tS+M%gQ0G9Uue!@|qRh#uzQ7#~@Iwjf{|>eA z!+)LOSuk5iM_OINi4RnKCcG2bSos?7`mTxzT3W@^*(bn4Ud|z0=Ft@r0U|Jv;@H^4 zpBe8k!>?LA4EUIsWx-Xq_4tK$Ve*c0l#w(racmXHnjq-n7`o;17^_sZtcMqes!o*F zL{~DmF2AmiAARmrw6y%9OCQ3al0*I-QM`yr+*Z*j(tT3eoagfNF;B^bx7s7_5U~%r zZ-Skj7lgaExpnbe>J?+Mz9S_*G(9CBqY#4-sE_4*5yiCu|H2a^v!+t_u(ZUz}%1X5B%p&+m>65K`={WyrbKb zt9{q3(>N-sF0Q#dZ~XcpQvj{Wv;ltU&+-;b#l=cpB50Hy#M^@xm>)~;cm8AEN4|ua zde*jD4X3j%ZH)TPpThcD@>`iXfR;3HBR z;PM$#(KfMMmDm$ML)M+(NSB}p*j#G=^-DB)56?Ka5xs1E3K8w*HR#%u*#(IY;@q*} zWH!L#u7I_s<{g-7$XP+f1Km7BhR3ZAv)P<~qwf zu|SUBj`4b+H#kIvZGkjRT3G8Sn~K_W|BH9SpPG*g-i4Yzg|(b<9)r8;a(w{cNbBmx zci;4+_;n75kz+c#62WRu;PHM)dbC3xgEOy0`Oub%0PD+xaaR0H`_*qZE6DDJapqhM z%gOObj!PaVdK22;5#fy$LuGsFWlk6K>*RK>B`Cb_6LIZ)08LVZ_$ms-g$D7HK_OJa zp=U#4i08_`4#OA;Plrgex`D511hGkvGbH-2!=7{EK=t7GhEZV#H``CUqcLrlNQ|)B z#lmB<=;Lv1d|df;b;DbCLwe8ONED0DxrCwVQB(U`|on)}qwCm4e)P_xBEf#cgz>@6F|+J? z^ntC*ttw1nt#d`1Q=CckN;}f4W1+B!ca6r2l--;13)?S^Z`KY8j-rgD1jo0Dk^z4=sr(6y@MN_0( zzj9SvvagZ00;hpa_8szv{3%+o3eLd*A6CLR0&#Ys z3>sw&i`8jk?{TReV_88=4A_vpw2r|H)$7DN?HpuNU zp0Z7Dax0U&yf~G4P-l$8j4MNx-Se83q1MeL2wb*@KS`a!=UnsilUNstIjqp9BEiO} zE^*lf{~BW0r1rP6U~N-yaR~9&6UljVy&Jy21tOQL%jPEI$B&KIE-8vKouxR!t6ltI ztwPv6$R2K##}=RP>EVc9KlmE43)VZM`8n<-tuc3XglbB6xd%SVnq@xuR5fGWA!iZt zX`#1j>Om%j=BfnoFGMW&oN=F#rbiC>%;TslTenNAz1 zkad?r0_j+GrjgZEyiQa>JeRxco;?|fU{p?0d7HYKY}L1D5EH-`6~Z-z=ynwpkodno*R34S10(s zaj|i|ZXP;!U7=N2hHTZ_td9>jP6cjg50LYNAfFw)QN|X8`Qg48@bn(MuBx*Ya%gNC zDgm1LkG!Gv)bwggOlkuVpDaO;aVD-2>L3>OCJ*P|;qUA~%r za%y%D`?#B~?Mdc-Y(^mk`NY#ou^Yxi83LEX)4^KIG>i9{rzhzsTku2Uk(hJ^Jis~M)Hy-SAxD8Qg z3X@vgfe|~Sfk3z>0uzU9f2fss$TSB(xX4Fz3WXKCYn+tPq?odHQB9SJu2~Ns8LFGKkk{SXhze{ozL)C`}ErpKQHF^ zbl-*?IW@DLl!-}{lCD`<8n|74VELf!(W}ICy^2T|J{u{k30&Vn@^Ir5?35lgSoQ4N zf$hsv{k3ohI^ideF5Hd)FrRTAr@5}V_({HSm7Da@lF=EK%`s9?;Q0*Y3cX4uGWC)> z2SUl6ngoGeijhyhl>oI*?lCdp5HnCa8+`xxhQrra$Vyb$Z3A^oLY-W% z{&1XYob4LIl6JUsMeK5a*;8@fExZ474`Q~)Gn;4cyEg8dzMFK7w&Gm(uq?NdW;Sgx zR4Mp_=ap|?+VUgT=tamT#oXu|MI+~2ObzLPaMUA51Fon3;fIfD9sc9B>b%AhY zT)<)iA?^4k0(lD`PfDeH{(hV;BHQsfQsD;IXjemS(57Kj8llv z5b((Jmyq~ar{ay>JQ!JgM3>mWG z`HB~hY+8x9^kY8D;0o$d<`V}$clsYpDc98$6uS_rqN~}(<`Rn+X4IMS5p`lO*zL!G zKAxEK9@~Kfs69id3BI$S_VmZgbugiKP4s;Y67duHd5$2)P-! z?!vwTCsTx9t)t4Q1BNVOlZU{csNWO{V+2n=n~4?I#2D75Pp(~=vASaBd9-K6|}_TX~Hm$q#0R45K zxCu*m_kZdr{*NQ_5_~F0(cblbz3FB=H6@yRoe?lp^A*Ts5`kRNqt& zT^8WgnY4BNurHu`L=8DiU8p6#vZg`)GzGm+E%5-%CKy}_nt?|xSFt`)a@xa0&O!*X zZi~vIiS&rjb&D#+l-bF(WvF3$(!vc1;M(Ug&H+h4FQOtyJkAmNP56U+ne10WBG0gy zHYplQ8;YjW`8j*@so~c>$#AN{ii_Ka(uq{a@eb=F=8^XcD=m6!lnD|TRXUKhk)KiT zeXPj|_LjJ-2piq=@baLv2i_6n-&zi>I@RI{Q`~?k|fDnCFAB6FwrF5-+wqm!rTOCGUhx~dl|%L+}T6V4N^gD$%X5o zp&`%m)Dr+SQ`p;8N{MUq)2Cis{(yrC)xhEo`2<%m!gQ`#?G1T#Le@FgdS(VURL`PUx-E%SDc@`yUPY zQx0$Rc*2ZsJqitQSO3FIM!8Az7b3Kcs|9vv@Vz9nexDvxsTZ8Xg*ZhcHwj`y6E_w1wf(>A?k zK>hK(c-Wv_YBzL{5|j4@Mahsda@{**){y75)MbwA; zw2&3QjOu%5TIO^071$|OhD9K#G~_enVG45d_*98)?eii@kf=NeyS#esCn1VM`WeZN z?b84aFXB_fQSIHdp6*n zX0lGA!PTVB&(Tjge5I=kA0W%gwbE3z#Q4I{YS(}`XZ+q ze%u@}&cf^@7&NZl#vDg?SyzsS2N`3zN^RzyC9L_c>D^q& zn`m#Dq_21URCw@aFCFs>%2p}RVeT#V_ME$amq#~1d2pT7pqGHT#RdnsvRXC!syc6P zC(K^Noojep_>e$4@h*tD$LAwYd65J3A{eu6v7 zCX+GNP1wYk~e8T2P%MWIGC ziNm?25euYw2^ZNwe9mW3Ie7tJdWwgi_@92ttfPZEwb=^-~GN>FPx^j|v}d?tbI&c98$4cwWM%1jPjq z0-o?>U!4lYP;%RsLV~!RSJ82KJC0}u7rBDglW{8J<)8 zt^LA1qL$ZYF6g9YVVAkH;1d61Kj6|=Opn<>j{t0rSyXDo`cKS+Zs`(wEM+fDz z1wH!yCjEBv!Fs;dXH_oX3h&@$YG-N2;5F3X*VfO_UNWHEXWGS)<%Qo!1!ZJ5x=vVI-JE04#uyrC#4_} z-|m)wpDXalu}e{dNuUam&-f1M5C){CE%+X^H18}?k-#Gy<2^duc>AbxC5u*$%}g6V z&|)Vl67NTU>=h9QUh1(y2RbU3mk|5d)fLr#wby0pZGFw7R80mGT6Hc14hml&piF(_ zrwN9sf@8?_CFH7|3Fy8=pFlL1P0(uHM zXj7N9vrZ#8vsjYyoc&}nDgSIwZ%eO2uL3({+d;Nx9nBQPlkeoYder@*>R9(@MKcHV zFauUs=)ebL)UZ4{GAC>ua%>r8ROYtG%!+X^(N9CW8XR5?us#WU2lzX?-5z}_dgRD+OzV)wVtmGU(%zqxzI^#dyd=NNA+ezYD0>S;(&fo z<>{%++dalI5UFCp-O`Yf@))ocO7N%?4?roJ9NoRJY5f6C7s5pdT$s)Fa9kijg_dg# zJltECGk)koV0Tl);yeRWoS%=upC+DhvM3#0<1y{f|7B1)c_)_Hf9=9;rgMSzY$0PP9&x`nUU%qfOr29Rj5LRV{2`u#Zy>(kD0TYy}^)ru*#(I z65X7Cnb=;T*cI$wf=bAx0c??VR;B2KYPSu`+uGm6lI~Qfw#C^Zu1-u1a?{uKwzN>T z?eBpu3K)tC&EVQebrZ-{j!&S7jE0k09Zj%x`LWRsi7+RP@Znj^PMpGp$S}kr=@2Q!DPYh+$xNcHH1O#C@S6$Jy7Ra$yKr>|Joy>o4reF zG8C?)Z1(??BmEtSnLxIr&ghAGt^2MDREkMGe5>86IN6W{B=ewoM$Uxiw$W>j<^K$M@jNJ&!1r8yl~NYAXn*CXhjLLIZ+X`Ux4`xoo&5tOX(B;g|Ay@7aoP z1e@l%&zydp*5poRgZIq|d3srEepKAf!T?j9Pd^4m1@z>L+r(rH3n&yb0X#i2#Mf*M zR$4HzcH}yc_i0MY=ZR;O;^U+zmIh9EMfelUnFcAgterDDkcZu2oBjUz3>+u%@TJm+ z$e$8!C-h3Yk}-2T_f#GOf?X`;gclT!?ZCs2o#5l;R%r-<$Q5$Ot|-T@ur#Y*iu?tfB~-J^!WtYy4tb3-rk?=LR0ok@IA@ePk0#y#t>_ zk+h@g!T%OV?**8Znaam`m-1QQ2Hx6Y&EKZ#q+EWcG8}2by@j!nN(#IHgwZ-sAJ8_E zhg_!3zhK zR?<1PW+_Y)B!~)#itBHa8sMbo&hH2Ypx^~CwF8byaQIiPwFtWlbFF*?-D>G#MA}~L zL5BB%DZs|JEuB#_=5ob6w2zyIC~zXF8p&|JQ(eBbR_x>Y^~F=4vSw%37NGYYC(d4d z{_xBGKZS89!`uIc*!>4;cl!O1?O@&)1eu67{FZCplgyfxxNUB1^Al7(+L7UTKwgK( zN2}jvrmMkroC0+1ELxh1!tRtwT6azz9@?)Vj!6LyXae&bQ3K*Lc2OH{!=wH@fop&& zsarW@I)d+xhqc7`9IN&D`>->8IFh9<+eP8mWKn?NEacuJrGQePL%!mN3OkKYby7;puPuT{5j&YRz}BUWlv|$Okaw=&f)jq#AA6P z_h>WEno>bAmZtbApu=qZXm{xx5i*Ho<@ZV>XWvI%!zXU^Vta{tMVPm1-3GQmf!wTzayi&N%N%%`%7CW2LlVYdqsDe57*AF=7 zqRUD(`c2!0%q7g@9jvitqqQ;%xZQ+2BCnBkMd52)d3+XQsLG%G;MSV>V}XpgF!ALZM1m|J}@lQJ)mg zWG_A|FX=bAR7*|uTjiBXh)`pAGn#OW`c_FLOsv( zg;9<*{+c?l#P>~iWUwj87*>Yvh__3bV4!sG6?iJu`0fXZ`DQF6KDYC=_xrK&vywaI z&XrzR^~ji8l+5cr2D()XmFp@Mjt9vW(1dFP0&1Z-ehQ^m{|C(Z-#hK}--Nk?iDXPd z75g7PF~_O!mAFF`X4P78d@8RB+4|iKD@e-xObFSUadUllER*!G;?|Y=*Vi5~C@OS` zg0gVowF)T2RAsD*)NZ89p|JRqjF}|@$3l8?W;UAubS2D7gd7K<-d9=%l7~RMm zd3ogFPR=sHyRMTQGn1Mvdwof&p=F={#nC2s9kltn#eUS4GAWOSeZ@Pc+ z*}tAY+M!$$O^ZFS#!qR7>+C&=ORkUXOQH0N&lyJt-6GtSd;z0bwW1%chSX+hrHNO4 zDPy%ykk>0s2#r=rhz4RHmf%VW)e$E0TVHskJuh~?FXE3un!LbV*r6E+G_+`NZ(Rk| z!MrgN&$10CYi6<{u(cj-Ce-?Kf%4z3*libjm1WdjU!I9OH=;bBA(DpG#yrIUngw#; z#K#FhoU{{VNSXs19>;viHfILfsmvuefC)4Va|MTu5=h7 z^>92bT%7`#`8){nA)G}K(B0!;aK1T{`f=w{mS~~W>zNCNuKe8L%zS0e;T0ztL2iOe zRQ_Hc)LjNBQlJ|fvANoo?}bnIpEZGH(%`HE{l@(%gIZ{6ia~mH?Y%19vqM$RbTD~G zrXh+j%dh1oG^vLslmF~G(cRcZ zBsv-;bLc8dW%0XoBuYMhI;F2IC+z1jFjwgN(Sz6ia%&^u!}TvYj9NG7LW;&azY1^E zVppr3?~S>6_qi(6?JkEuP0_GAzE?zXurxk-?>_=BWQVbfI#$+f&TazIip|+#-B9AM z5~7lCVs|swX?XeQ+M}SbYbLunD}Rz1Wa5J@y}f>;Y1n>vOUN;~?1{tlGcc7kQV=fH z!yO+u0mauvo~t0mG!8#5uX-cFK6QIYnfXzx+Eq%ZPd)?6;lEMSAVy5KPBhD`#q@WG zu`Z~qkS$-qHses)A@CsT@gV0#V=M!`2&W6S#1|d};@MsjgOOzf z-<>Yqz#YLfi4Rg$ZlTw%(Q9#(Dc?CsCq5B+`z$MdHuY?>yI{(di5atcnkqYz4B3HZ zxX$3-j=oN9BJ?lh>(zaA+h+jQusIrg-qyTT6$c@uss4lAXE{XSpPFT0qhtHy`ZZC$ z*Yyz>ubsx96A@H0NjT*Dtk7tqPl*!{$sGPwF(Ls47v|fqx_<9SJ$-mq5t0iibU`cm zJwMh10=qi7A8JJzhr=FQ7p37*k;tcftimr`pkv!IeZfkKraHN^PgCyDyXO~iD0DAc zKLOfcy2tml@kGY)J1*aJr#|s7Mkv{tm6~0`JGZWMhmegBVHxlz=}V$oS2)OiL=?d~ zia?uHZhh6@=gTP)6Piir*Tex-*{dsPpc(@bxg-H3^51#?Byyl9v51Q>CV0V$i`K_Y zF@cEiv*8o5@A4I=&BJ%Xsa_!35$%xW@dFPq>3hi410fj%EPDH7b92LFygmtaa zpqu`M?Y4yvwftl6-$@kcYE8R#4?@MA2Zk%N{|9mf{nFz5cyuEfmd;v#r*k>&Rb_{r zjb<7CBv?G~Tzq$woP8CzxMDeRhbm~(A_3;*C;%$7#unKCCficQg z?1q!{)o^_TSvh4nwyF5QfDnO=H^x#-3RRTMI{C`WVWilGRkpc=uHAi|qO_xAHt=QD z;<>Sk(a#y^i_ZHNLKGK9ctvNBwtznGgvNdOIBL=5J-q1(O}IbfTV&rgy7Flrc>h%u zomN?e2QB#S7eR`05g#3t-V9{Zjd&V})uhN6RTx3IM#9;0BT7$|^Jw_G`SuhMsed8o zz!%fY-0=K_Ze8%K(MS1gdX2~_h7GDpvSxD1IQV0h zZ1LSK*hjF#B}M7e;r#>#MG8;2((&H-ihP?S-MUg4?j%-Kr!1WLD@Zn#`9H>*Nj|vau zZ~&{4kN8U3>T(eGIUB`gmgKs7pQ53hLE-F(eSz0+cd~g-bf-0sQ^1?=8@Zxmo0ImE z%NDbJv^pC_C6F@>d7*qv#7A??d%85+j7}EuX+-qXEUx2B2{uQH<(n6)!+13OVW@@g z_Sz#`-X~gmU!%5W`aLymg((6P&i~=&+(z45(>t@F*<`3T_tsO-zQSU$nm3BMA6-Z> zs0VVV&c}CyZs9ExJ89mq>5Oex_0hrDiB;uq<|XXnEi=C*nJ>-Thf_Kvey_ciEr(I}z4mDx9`D*ZDldjYx{Vyj*D(n|zJPf1F;1&ZsVvi1 zOmhFX>+G8I!}mgjTX6y|%l_Lri%mgI-~@1i6QDr&-xJ`sNw=WqMq(BYcQrOh*Ntc8 zdV<#O@uGU*9zU29I&Ehc!m^Wb5WM}r3j;X<1(IZkB;y*Kg81OGlONIzJ!tc0v!K4O za`Sl)MYQ6xz!imUYiUcO9{&6Hk#Qyjx`TSB-uHqNQ&)a!YPCst(H;b%Ont<6;EOA$ z1#8XKHHIqeV?G*@r+So<6lF8^OKK$U%(UG+_~X^HiJvT{``o+=XMhl?ZHjzkgeW;c zEf_kG&muvRd(kY~OnC43Ztd$bsjv|AKm>IUXXoE0$otrd=@=(RB%D`?3)O^glH*0Y zE)b2!s*V}JR`5xCo@XD$MU%<@5JLaJk?VDezf4H)EbnL`q5jDy!ROd+gBa8L%nYaQ zX5m{Lsyf1vaYu$+YN@F z6*Sa&aC^MynKq~gO~;UI#~)Qn!6intO1D0@4~9ysvHqvnC$M2LoeZ20gKpv}+!3kw z%4@5evW7&&o-KvTYEKs(nndXUt<_Wl^>>_PrvcY0Nn)q~COo>$IDJ`#tv1DgIi}+b z2gi%w0sB0k3%eLVY%uFUGie1EOn2F%bJs32mLryuGtDm~sZ$;%ysK29Tqw0S!zUy= z7OvJB4U9|G%!w6|0eqXaI0E3LL8y&^cF4W)HgGD{1h$Bk8dBAVhbKlj!%STk_5`kmAz9$Kd{ z5JJn)kwghKLmoduAWz9GiRV7oTx@>hVb6V6Kek?1$d6u$SIYvj^O__`_@!v1flqu< zD|(Ks`pRpMHJV3>w$_G0_X&wuQOSwE_N_?e0{FoC`eT z5fKi9$KD-cpxt~57aNy%^)B)fg@J0A~^#0%$KDPmzN5AeH!jH~2Bz@14yc`%M zDaGU_c)zy~Viva}9k&Pxay`igS38&){Ld=7t-0* zx9sI@dbRaXbK1RuU?40da5^jjvsX+9lGTD}I#LxZ`EW#L2CiqGBC;XDOV^3jdg}S% znWBr=AieTRzgUO}V(r@j&1<41N*h<_o@(89RfL19d&Pf7vep~h!H z8p?&?&!yt}+Khky!4y7pzFSf&ZIQ|yapC=@CgoMCXWa_yn1@7gi>2Y~0vaO!o#d@I zH5THZ8brcquKm;%8L}})1(m>4?syapFbH@iCnpb&n0*88<8!;qUVkAW*~C*pwg~tI zm79Ubs)!8Bk+R6GPw!}6Zx-gT*;AcS%A^mmp}COzfh;E{bq9=}E~oqyeNIQb~JiW_}Ir48dp2qxDgk@ z|6WQkU`L4)MB=6(<70jT5+MD%@5x;(%X4(TtfyQyYU&Zmy)R4|X6+38`u;7(e!8A3 z-5ruKA#%<5+=|PHY|}3Z%9)q6#Zb_2sEJMfSyS>_tAYZ8EncgurWrQa(F~B7`MF!7 zj;`BfOS!^Y?uE-wRpx;(De!#aYquCkUVQTm5ES&jCb(7i^7Gz*^sWECHaNhlpOKNd zoMYhG11RJyT>BGwHjJAMi|%RB_Up7j$kce3k1Yv6n^!ots+8f@XjxSsyu5V1`c)}y ziZFJo<#*3S1UiKL9;6lRzzCUwdNe#^uP_FDI>A(43nd zju>}mDoLp~PRJ7tmmbgYRydqHmOCZ|l_9r>k-$4UHi#2{7Ti!nI2~>Ppq%_Oe;Awl2nj_!xTzU+%-?#Pl_V-rv-Q zJ&;iyl!CnHMkQo16kvs$F2oj}D9x_jWp9f{jAoOU_sSlYy1rjCGwwXwoaYA}D!-)> z`SPhSge>91_^M?43~WUQ#}HLDMIaXUsa~I)Mf^E8xyl4YRp`&F#6ads64{-M9cG!v z_LU#StER>Mk&5Op>gBrDv)5A^(2gIC@A?GL1?J}6zbt1Q|1jRjB=@RvVYb653X}r( zXG@|!espviB=`0X>RrX;bnzSJgG`>jpUX{nLm`w-`_>cDSov5<*j4l2UkmDi!}-xG z&ytF{X_aMK50#i6^OQo-f8}PaNGpfQH*coL#w~)U%^2$wv}ak*nsYO1Dus&zsFEc8 z+8;i4nHf!GnlRf!E$Nv%>(#alv@aTSKUC6^VT=bkA;-C=H`iPsGYbX+>$9vwDyokc zPHwoW`Ut;S$(`u;vXK;ZDvn5_s2@;NAXn&tizC~5@W7bme|J^zm1C#hoCP00Anc9H zzocWowQe!}((c+8>0Nf|;hul>I5%{j_dQ$)`E(A6NyD5RxdBvrIgBev6VKDVF3$a5NergH-G6DnWdQV2!$?rV!je7g8lF*1=aSErS0NC2mOa0= zzYxmx?gKMC>uc|cbYLDN{cuu^V3satevm}9s7z1Rs$yG&`F7pW;`y_l1Nqx)?&^_W z7?#!eZin>38C48Q2(!nx#;Rk#>v!qO*Ymk%{zA;$SDWnDfkY|th+<&Zhz9tazYwM% z4LbCTeO(%7(bbZL@UAs&fqMy>cXx_uD?p);5`2yF!7&#|eb#()Ydhu`-ZylqQ;k~< zTCe+`i(rf+g%k4Pfw*@fy)&nMrzErZuQ#4~jw2Eh3y&g!xpQ(R`-N5tzLq6kb!IZ* zh>)v0{R4N*~LLQETIJ#y1Ol&pd#Bp?mv*o;{R7_)I7a^?!pD{jz?US5-Y1t>XB} z0qFL-dbkyS8ynK~-r!g^x;cMk!SkAOj(zhvSlF@Y8q>O|$9f1~m7eSpjq+pzhlv4) zVK*g@$J}J2G!_KDG;4O?9A~xu7ou7134$&j&vx8?I7qqXxU|UJeL_1( zIZT3_6uQZx*;cbmYu`RlcMiTjMdfN4TU<|FNb3=o3+T%G_!@(#$@;lx$oz{ zCYQ07ICDHTA?Hp_bL*Q+Obp>d_TIwPor`j`K91F7l*! z|D@e7+PkTKg-M4byDu3Yq~V60qh8J0d}X9@cF)SDTS#6K%IL907h-4j+iznVI3`)Q zq4(F2d$xv@?-&^j>G-)cnIWo>!nSBXFf1^ww+r|d`kNvIGNr$0&`bZ;bl&-jHuBCpq?kaCQ-EQZ2IN7B0Vg45 zs?CaJWPj@SQu=I{fG89(Jsk?c)U(uD`5oW%!dZ7(?{C2OYz8UA;G2(8>4GZC1!plF zDR=%tl=dYGHG~%`F*IDPpxiqPZ$#d2F7*GqC_Oxwy1(KK+xP45a z2VUdje0q>sHWmHfBHE3Iye+9G1D!dtCO&@krQZ=?u{pEH!EItd6*4kjOS)WMHMU1% zUSj|1*0$4zo#pfyav*w+$|VUQd$sY_Cy{07Jug#xFl<_~83H>&@`Toqaw?AxCJk@b zEtNMIt{Q0?d6xSBG56l#RR90~_z{v_GO{bA?3v6%C8QD+nWtkD|YvY`ybXNU#FSv&tL zB_WzuYwxK%eajS8)ZWCGJDEdOSrh~KI7|UnjtwYvd4YaRUjC5nKUh384Pg~G3=2)Y zQUQqQgoGzB*B>fQNv!VS>DWONPaTYHo zB3J}dBU~7BFuZAQ`)M2Ue<7E*LA7gcX4!kh#zf}W|wNbhTiK{xM3jGZx>-g7{=@AX5b=_Oif`kJ^Y z#j|!#-;L7vZ@=8@_7jIC&#;3NRS0z18WVYsBzmdJSJKM-cJ1+zg`WO>GQ+O~ee5oE ze)Gu{q=>299*8*V#|W+O>jls(*Io9^o@g**7$n~Dqbq5!46Mn;vX6bd8kPFSl`geO zFlhze1-1JN0b+C4jzC*@_3l%$dEV3GxH&f^OdDD@ zZiW5`Sa%Pwobxk8k#6ZHa!KB!YaGr`q?%hYE&V3$FQ*W5_=xcKGSoQcv8mqbYb2v# zT`E@?$GirU6kdd^UazpSYYk&Ro@P7q4}PM2{r)l(^c9SAiLX~DNP z{G0kS`=kK5SpMJCpYG)V?G{olh}z`g8hD@>g5xT>ROHp5WW%yKbJ1Va$n(xzQQ~~E z@C~sE1I?sXb`{;@c49q@DoXt2GzKcX#yCz+dRQ?RdgA;8m;?m~z_hJFlp+?e3_@>UFJ&xJmQ z?60?}+yJj_@!zjKls3UfQiU-f*Qb~NLe$F9pRSYdxz28*%n8x6xw5&NM*VKFg+*6> zKeKvFJ?@i7cdkcip&VX`raFnjEXD&XIjWC1VPo-W0ltl`88&Xet%^&?%Uns~E^8@I zlOC_Bc+}Z*Rnp+HIGKxt$|rOVeAi$;hxmaPn4?kiT>w(5h4^d-1KS+AjyeB)0&Yb4 z)&1gqz7Lz07FU-8R0XN1xD1BM4qd@|w_7)n>iGTz^!Q)7& zYb#dOKURqId9ofxm;#0d)iQ%;NCw}T&ft$2zc^R$M9HR^u0=~v|F=5@>h?tnpNLf= zEl#kpP9b4QoX$SaL$;Eh-;uL<<3Jn?}HvQ_?we(%wjq`6*CqJ zqPaFRmiZW^hxs0VA=-%m&nQE-Khju60LUQftTf`4tVh-4oQN{N9L{2*3Zm;Lw`7lr zK_Ov#ZaYNDJ)vvfPgI;^G@dzdT+U>l^L>y;#eE`?04$^cTO$U~QU)w{7c6U#Y%vq> zL9ji?r_W*zT#s8oAIr=M6YO z=djkWEQlH(OE(?$o4_2zuEw!n5uq~uIcRxILtGY=UEzgb7K;fwX@`EWMO*T{NB^^N zRS9J$@?&99WiRsxZ!|8>IGnSJvAZfl^>E^n<#EzBNDw9Z>&S_U{rkf3byJGuNDn5B zr77*JTin$8=8W^g1NFpQKrWwC;P-ab067ktrKI0G$(y{kZHXlSoOG$h1g&_Ifmw_K zYMZqhq>xgwc_#G6(9%sK-(`aT3~C@^A022hhf2&tHvZUo@H27KxMoI?8#7Ft@K3Q( zUZ1^xfEld^9g$%SfIvRL9NpxA?CV5i&KeM}dDygCOBT)f_lwpIjFfK+|Cx%ZTF$o5 zX(}U}@wLc7HcCBSR|Gp*H-s-t#CfcqD%gJwZj%CI^?7_gEI zr}FmJXwO;qi#u!QE&-|l(&o9rhz4ZYXv0D6(D%T8A zx}phoLve((jA_b;h%gyOlRKOj`?udFJH#d7sC%s4F{lSL- zmGp=E6~N!w=6B_i?+vu1GWP;bGkS5e;)q=x0aIqCEX|c??}MOVzX0D3G6b`C=D+r< z7Ar^$Kb5@w?uG#puv6)DNQM<}=vyTy>*vggrk7dcc~@NPxwWNvw6?V5i_Rj)Y3B;0 zY8>a83@_!~uu;6h*Za1CXTDk|Y7&r=d98w+@h9qp6~5~p0M$~W#R)h1E2J(l(yeuP zC|8HZU5lRg%ZUn@Wm0Z?fHKj0H7tm;WBW=;eg+wT!b`@^OU4E)$2 zaS6%bI}!(xyS&e09s${T za;)Sl1Ks`a!(5CN4^WbweePs|ZT*;F{pXGL75CT~?z7AMv)(rBg2UBG*U5j^XybC~ z%{Yg}WNM8~pVn3=T4@&kd4FS1B;uamJ&qg-(NQ6w!SPCy4PN_I}!EXNk z)WhE9Yw*qjZ_C>TRwjPGhcpf#oTcbvajVe?Ldx9%1%X%FCa>i&?86e0RS6xB zHu32869v!W+4&2pM1JrGZ6E%WG$Y8d&Fm-vL^;&nfI^16-14dBQYu|=9dEK#Z-d4A z$xAoAq!5P{$lBkM?*=}BY z&CkVMkb8&m)hl>97TUsi&U60A2TBj8cJrX2=3on-H?Nh-sGp?rwP9yp6W$Fl$R~9n zcT9cdNSi&Rcx_eQJ473uH&iXx{V$hV81IB^^Yu(sfOuIDlf4Cx$2O9FW3ca5XU3wF zx0L}CT4ax~zvtAF@5i2lN;K2Se{`NTNC9QbGGs?~3%pb-Dl&Z`pXL?MZ96oDpSR5C zV_54{GrKjrL(S3%{I%k#zfPyX5V%1s(=w8^!n;E@fLtFJKcRSb1nI_32WJ!D>Sg_} zwKX?IL=!EY1y99**R&Rs`cK<=%RQ08SP9Sm4Z>jhNCbLZr5m4Gxqyt{*>06ib&kTL z)7N*6=i7)M#4jO7t#?6?sSNQG#}fQ|zy0zn26s?~NJhYa=s5kjFo z$3Y|a)op`npZqMl7rU@+W(}eAR04!!i_Ld zZjrTjo>V8ex?Z*UC3GhLJOp6r0Si?x24peTiklZ(j;!+w*UL+cp8u-2s(AOs_EiZh z_LqQL1C&7OzMy}j-uep;m!iW}xZ}F?s_rQ7-Ruu>X;|=LG1OE#uSP065?7^khOrT8 z*_xr0Cbn@VWd%Q*vG>p3Zckr$wN>-Zx!b+;F;!a{wu1!nu#8ZxzmPg$hG_8UK*X>(0l@ZZ&@@wq3fm9& z7oA!SeOzhGdCr`*H*@pp42L3*LY&T%9?0u6HU1;UL!wSsQ)hH*BRo@-oZ`O=)i0iR z9pr$VsD1hFOO|W0Mk2>fXCVcXy}Ys~OdcHH-tq`Hi~%r?R7wQMh+Yu>LUgP8m8j<* z-fvXn{*#*Q>P-8AzO4V6)h#6+)!C7y&8@0$rjy;y&biatN=24&8lNH+0Z(rae-h+G z^$SpFjVEdq+`+lDJup5TZ_|xPg zQy4a6Fw^|-@hFZ=hRnQJ2|m|_WF;8r^`fofgnd?`9^e*ScFyss%<4}$!#LdFo3wL? zuzgXXxk7Y$`L8EpQg6STpnNZOLwpx!et3k*ai?DG2l(~2YtDxTdA<|`UhyVB>%S0# zAz4i{B0FHbxqR&|oL%VGqkCGNC`6GbA#2}JeKZmClYjqLAxRcY0NKrR8^O5Oyp=|F zyK=N&96Y3muJnq)<-SY}rPJW{>`_zYW@{E7OE@;c#pHck@t*p6vtmHDOZ|0$DZjf& zJquS=<<+6AF+82!N9=OPb%4UaMIu)bopqV^cNyW0F`ugmR+muXeCvo zCGA76GH@r#1*J z;etTY83*oT(_&`MWD~;Yh5Zm(Lsv?E?%chY)fpFTpm=|iohm-*Fl{z8LbgzD9vz~d zo%1@=amip`I^3F5m$32j5I_ZXF8HiC??0` z1P%v?wHynslgtn6uJ*t6;*4m!0)krMH4OENJBb}Tocv|1CeC=Y{ySes^e@PoGb~9G zu8^YlwRlY~5*Ue>(v}x`ff!v{Fh|yO8Fr${J0VLlb!VwnUf_KIZ82AT1Y~WkF4K2>UGCV;L`pk(P6b9S^#5dhgDmMD2_o7d=IL#oS5(7gEf6fN3d$GI!d?x}6>FGLid zyq86ntKqlLuYMWv-1U9$a_sthGRlUhY=4zKlV^|2;C^xBg8`8Z$F56`FE!5>cZ{c# zrTNkTC3Lb+^L`wR=LsHqeOuFaE9xqIyf~zHDPO4ZDIcQW=bXgvZ!75VH5Z4l|EIIrydq}I@afs(mI|NOEP*%;JiHDk9$hAi zxHG)zxVgP3$E5CFU?^a&4SD(k?L3p>-f?7&Q6?#QCL0c_2oqw($~Y5@wL2UyKXb|F z(p(fem3jjJo+QY=C?o#ypn*Y|qbux(qr@%g!f()|u9d71GG!@mx8*E9N7%=4-iWep zM*XtEy;{9B+6kK(=G2!CysrS6`DGy56LBK{UzGfeAAI4*Vl!)I*9XN*mEHS=IrEIH zw6k2VmT<-HL|uq)(VB&h0hqW3&N?{{?)~TMu)C{hTcUjo_h(HZsV0ae-N@kJW2$gp zKZ2lmbD@FGP-*ugVbR;uWbmt#XDmG5NZ+``PDfi&OjRzw6L>60Kq(PIHUI5mj+r7L z4Ord~#~SWAZfVsYNrtn!{d7{>psfu>| zQFArrzh@a87Rr2uPx5k+J^M2!Bm@rdj~hH*>GYOpXz`6N@bBbEwoeq)#L>sC{^lpe zH=b+C@)0)}KY2t3myk#vhkRso6Pv_VE(WrfsgkSx zr;n;` zNP8FnnV*cDE*ZKQW8vqaH&ny;ZG6ojZnDYZb@4aJ8lA3sY-F{~L=JK;sgK~NH zmuMfc@pI#%1=I*6OLkDEbv7iM+fTzX+Bc3CR~rxbsxL^!A;U^h^?Q%MYgKVdHmu1C04tJLxO~ z;aj&|#%APL(s477^pNalHj}cTna7{EQiZ>ockmV4Kyv@McpiaDaKcH9@11-t+!>IZ zB#^vbuNIii&BazQN*Y4`ah)tnGaSAT3+-XdL|uIC{neJD3}-u<@Kc1Fu+Vk#7zpu^ z)?zQ!cTZ#d^FdK1H4E3mb-armwv+#ZD9hXE7*)y;s#{BS-_hOTXFb!5`-so7rdP*5 z4;n~LUe@2GrMnn(4upS~-*P~OzWBgH*L781 zgFaXSXtpJV0%Vjaa%LFR+>eHJIN-~8=)>1*AeDrFIKJ_7;nxj(CL7E&#c6#;eknjD z{tHHoAt@9(5KtR)zqY6~K*}W;Eb0r|ZN%Rfp1r)mNJ^N&FXh8Y|fMy8cI#i2bEzFcf5n-vyW1ES_ixF}NKh}72WP&^TM%jqYhNEov4|-pn;?zyoWn7O z<@CBeB-gGQOfQR`O;XexH+))hhM|Sx!z@6*cVn}} zm%czC-8YRp1G#8m0&EzglF-ej+E4py)a)D*6Z-tV$rnET!aVuyX`W!pJJ+;7W;j(| zGN7ONazPsvW<=gSSF7JTJ-#EHXH#1mcW#K!&0<$7K<5&~1t|^l|7UGY*ji_}JFvU; z@#{B-^IhHh3u|bW&lT0rB@fc^DE3nMsQZE!rB_>+Ih@M?C}ZK%D&IDy1l3{;DxBVC z1xein5^DFtB)5*o^V`9E^zV`^TaTm9AsKZ3 zAgI}WFWH*gIj1J%A|>5G6%)#%<%k_KR6yqBJ1XxZ z=)z1}v};4^EhHnsrSq3xoD4j?sJCND`dcLp*(C_CZ~O$;;Z((@*23G$@FGze!xsjW z3(vW=#k?Tfx7xSgoK9A8K1jQZH4FSYeW`mWBn<(-5fzu!FZ(1&b>T9;u(}&NoX_(U z^emr;2u6tB?iVhLZWjrA7_jJ8 zL+R#aL{xiRBM8&m_)Of-$@{~@Ng|I+;yGj(TG*7%ctp*5)Th%TOQ@PKETcr-(%~{w zCLk{Lec;Rc_5-XE<*(Rkq4XEDfP3~8WO0YA^CVq`gHQ)(>3(Mbtd+Lzk3s(9yVB2% zmDWoGBqr3xo45g^O+5_7u^g@Bl*)?}`26YE#A$ffh)*z~y!?lRW`Y4;NlG z8R40WzBfnq{BG!O7Zu_gize;eSXPVBs2LDR=qO}1YwYdd zE6MczR?h^v6F(3;lH%e*8dOmfb2naz04VKGzoEFJeau+Tf*Be|)y#Gz7*;5nu%zhf zMu*SUb@!41DvK)+fB#|pssUc!c)_zb$n0A$!N^7 z<1YjilKeuMcGJNk+q1{}(AM`Bu4;95n+ZAl14+7uxo?}!_~|dil9AjqvMJGMD<8d@ z$l<^yDNv(3)!KLeS7DsH1QkLVUazEu&1qB6|LJE*VjC8Vl8Rj(8|zY%B5${H(>r$= z<=h*f#&RSMV+Vms2F?QRff+65Fs)1C7pCyIAQMN)W%W^lUD=b%hHF2a!FYD0iI-gq zC)ZTLrouD0A?kp5S)1x}Y$&=w&6$6m_IelB3$h-SfY>SjVO~a}?l%I79)7P6?WV}S zsdwJtrQH;}^ykj>ck;7)`MjXvY`e__H2o651O_aSB}EeysT`v zq$wObFEOYpuV!TXNZ&y#F7L@ldq2WP*l3-V_9xURN`QZEPHL(P~o+=UPygT82=hEDu-BMDms*!uG_u*Q%kR1+NJkdxuFwNozM_ z7hjvRSH6HKhgQ@O*hQw&0`auUEF2+MTC z`G+3sdHxVJ8rsjNms)tlphVdGV!L=_2F?nSFYf$J8c=eqLWQRa!zeu2^l`LC$t9K% z`sXJ+N))~<;@MxOIrLKLsQcbU+B5PXd$=z`_tyXf&(w!)V*!`^j1Zb{?+pzT+0@%P zr!vBf|E5cG_SyALt!0sO^80&>LP_M+!-bprOUvF3G?I;>4yP{i zH5!CgbGbEMe<4aTP$DIgBo?5ofmgxmW9E`|&abIl9&nOb#7MMJ$Z1-F z3~=#0qdk3P3(%5^mgMcQaqI6%Qw0}btHIn~D=78u`i&lH@n879oH}2lV|bFMWO^;% zFKvH6tv@f9uBH zeh2)kPZU<;P6N5m9M$HW=R6HnQZYzgyp~=b&3?1U+LEm_{pa0J`)x|4$Ki+kD^&QvdXO+e0{wz_!sM; zyWcx6$3Ib4Rmuvn6J>rE%&d-8eeYBfTxToG=vBrQBaia~9jTHJ&TonshU!VwUMHW2 zf>o9St1Jglp#NIsLZl`7+`2z2);jAN#zORM;=$GD>7Vc4XEAUx)a+c`qe{Z*5Wwbx zi2OsyDX`b)IGdR}y2en_BpUtmH)NcRzLdK#ws$0L6-le) zmb-Qn?@Z7(30{5`q@B!e9xgrg`8_51;7Bde?j%D=Z4Si)vo+)TQLHI)F2Yc3L5dv) ziIZp|ect9|_jbPnU7{9WI&ch#uVOlVx#y+SS`8k4MAZj3po?oR;Jo%x zK}n&}h&>#>P(sPzeDkQG<>MUT)PQX+u7%XHSv~(j6$L5Dm>Kn_f&q>xv&fm(WP_v* ze+W(mwcxXfFzN&Iy_YtoNBzoKgBRT``Ru<_1TZh|0ejzq?P`4t#?!d)B(5RxLNp?$ zHpqw5P%t_(?w-}*hewxeD0tL(J_>?t#0D8%-G-{s@cQBz(cfotp7WzknO+2oCSAeT zl(dgr%fpr1;h0in_-hn5oWme=qjaU9^f+8(*X^R_7Oq&|Ap~+pgSkF_F znGebVABc@XDZN68j*kuhg?ryzv=zmu$k)8-k`ADtnp;oe8`*!zXwX}l|M~v#CHm5E zJ5FsON%Q*mj~`8Xa33v1?bNZH#NP&&A}7|;>~2mRf&d~-(b^j^(W+n* zb%4$gjwPVycV}T_Yu-l_e&38=WoC@L zei{fTXBOFy;}c!8CwgcmniHK%^-9i8K5B(*9w_^Gpp^mOEgiQ_#P)*z5J0yOs_`>p zP4lE29EYQU&PbtXW^IZ3XKT=@X@3Mbm{Z=&$dK)9N2;(879QewGHj_E|6PBRslxhQ6 zh}IrO%qihhDL{b_G|R1jd2aX0ZuN$y-V$q^Y9{Jq2swX{w{wbqMHBD1Msqcw(eM*& zW>t7yZWP_@{$jkfuLi+Kso!CKj$%joI{CWM|6)1@C1>z&=uMI&pI{5>Qm9_evC+zG z)GI4<#zpvt(tB}R%m65*5d4gte>ol94`l+>YpC!tzFhb?W%TB%8hg^+!-$xG*x~d( ze>Q^P-ND(x*BE*`{K5B|gsi6>_gbZ-ymwW(-vOjKc^A~4*#k@gzT_z3GEN~&rTld_ z4Y1mS4%{`&Q?V-%^VC-=qXk0Luj8si^z|gvsB-zrUfIlUH*F#3;|3lfN1+Rk#*7X^Mq_2YIBkbsYsTT7vOiY+*t9G*AEL>lG`%9TDp2?kLOD{W4X`Q%Ppe z(9O3|e<9~wzCD_{&Gbrv4l293Z~WwY>5$BEB=@~(#bm{|l3mPigw}jGAjDGiH$X`A z{)Lk5pGQr22~I91B_e9j-)+-kcDZS8pfL$?fNA|g#}I~gYQ zZFwr$0kWq~lpu^Rx(AD{?z6fqqu-*%tr7Qz4@b3wzKpI`k@swy^G_@sd+xcc#W9@6 zY!#$-rXAn@O4Y^C`Dd<~-zzEV-EO&g!FtVoP6HjP9*1?I3rYz>`qHtx=lS^qM*_TG z%aHBKHDqSjz$kK1yIgnpxp*_%hh(zP?hU zx*U-ZAdiaB-0_vfdEYD88dskGmfOMKNPrkDeWGAG*T8a=V2ih!mkuWw zh%IVF70U}kW*<|vBYz~7_&W97&%0t?#UI$~#UM}AMAxpTU~__$Was3I1h7ldb~#nC3n^Dj!3-JWrJQxd&rmW96cvAC*~|ItXsfK{|W_`(O4g;&o$gn-Sy z3iKu5Gf?w=3*3{(FLG@Atxos1*y514vo0wfRKc2-T)O9!guU1xeaN~M$ zgB<8N^tfF8IKILzo6r8Ho(U!)-t2bxd5d0F2Iu%zXH*u%N0hyYZQi3roSt|qX5iT0 zZn3Hwus)YP-r3+b2~wp^mC_N+e&t+`Y|`_*Nb=nWan@NBz??m%s_8QsoR-IKn4RMh zQ%Dh1v-SN`_2l#9f-Sel)}j0URGZNbhF}jp!;p{!(1PR|K$7J;`80fI_>eiA^G++nt1VKUQ%Nv_sYp6CEgAwRV|S#&UodU#(uTH`5p zI?dJR)kM~(cR|aSIvUnnAikW~aKk!b&s$d$_zcQ!ktv%ABzVm6(yiTj z+*?Y^Dcl<(DPy-wsjMoP>xx@EQ0ALRz6G z{5IAs!*u~Bu;<0r;p4?DS!vN8%|`gRz&Aj#2+@QPN6UszQjHmxI(UU9DBb+ucJ(*Rcy_1$EqTV0hE<&XVM|o_d84jJ^8M<#4O!hVtH~ zG+T90pZf#XE-_0k-t?lL*|cbXW_y)uNo@@ZP`nsg={c-awYYRmUg)i~i)7rbc5Ylf z)z4D{NF$s;2w&)CoFj$Fil2Y97DF ze5#wnqd=m*k2^e$TJH*7VLtB5Jz1z`FZrXa6O58#HqCEQI?-*i9;Oxy6*g> zLazD3t(8}jI#|>u<1FeT9AABxK6lK-^k)0l+F+)e{#}*w4a#Rddzi^Km<`Briu0Hd zf3DgTqm6+skJq#UgvC zFVFjL@#RNnUMVmp)v=-bbuM~VG(NDX) zSO$!uMtj!JqbZSgmezLf7;dLk`NOu^1!GzhN6vkc zHO9zQ76CNV+6sX0#;xPRBs#e9AK@U+`;~lBLTiVI7}mF4_g-c zZK=+Nsk!H`S6!}W+Lrb@g%OC43lqx9C`>OIOp!Ekt{Z!N*Gli1o5b_Lfdif4y09X} z`MRn(N2-RAC!SMRlkdL|BmL^+&T-1nb&>My(T%bWfXV^T``8b5&RIlrZP;Q5#o9++ z&y!CfRE5$cR>peqqYK+mEIpL42~dy?C&)%45IbItuaDJ$XE(NR^j_8o+)Ysk@=|t< z2dFIKBKrAB0!#8jT4+Sx75&9x^7Lb+H-b36UG#USx(kG}YfJEOqHwm-fTHH+pV$OJ z{#hP*S@awNkTEyL_sE6H7>lad4f4m`{%HN!HR{NwVxS+WEuDPPO+m?=gLW}k}a#8z;dNk zJyc?hk#7h3!50I|`B4-(hoGx&FHdq0wIHMWN*Qshei^l+rQCu3XhV2|rR(973))%~ z2%jw1BzGMo2c(QJyzD(hB%5gI2UDpRk8ZJhg4vM$B@9bn{5&d(JWtOpvMETi;Zqni z7X3qof>1G6S06NMZb>u|?qeKVzU^YsL?&>{t!%K>CN*WMN!Ev4jUeVn`pA?GJHVjj2n_RT`W zU=+wX(E$TZ?7AJ>=Cx6AF~?ZG@b=4AZ=)-erK9Nfx#6qc5aAA_{d*e%U(tqMvvSp} z4dl_=k;112?!D7SU5*}14-rD|ZAwUj^=;Y| z3qdM03p7-Po1oF;owUZm6KG-OovUjdtzt!0M7^2>SaRa!!206#$hFQky8SO$!X8Xdy_!(WUh#l z^tKob55PzRs=WHTFaTO?GLbu>2C&Vun3HT20UiR*lL&Hl1Y`!iR739#I+55BdSHUU zr!;_ao?Rx1V-B;T%}bC4Jf{GhZ^g2cMwD#=syR?BKm(5h9~M%HIAUre(Ibx^Li`NX zeSzhoe>JlwfFg4Ih2WE*{};L#)4AecLP21zfO=kSHA#KJb$+v4h;qUcp3IxQx zB6|l>JIdwR-&$*UFZJ$eRFGotnLYqWz-nw7fWd&DCVwGC?lVVMgxmAC7^5-dr7%uH zf~Z@~-D>)czKzP=Os22L5SmWfqJAnWcB&Y%4kdcjM#GodKh<5hv$SDR5ggZnhfzy? zrjs-(n0`wM5AvZER50(3Hj(=;B9aLkTw4fvRsaLftUP~sfs@KP!I=r|J0Pq6Hxnh|3Iw})#?k>4gf%at$9 z$X6!xd+IxAHYZqJQ@?!ix5O*g-x8G2v;N)Q^oijGV%xqJ423K;sff|pKy$nMJ7?#F z&r6ZrVsCRE25s>PI7wrZmei1T{A_*s{xKjLw`uqco%c;(bIjAuT62#uXQO3HL9=R& zKXKUs(v}C8m!rIFh*Ap3-nAJ5=wWRRlHETbJ@9!1)*Ynb+S7F}(0y=p@H8(nn0EHT zq{dfuhQW^=pI9HZ>d2(N-4+O2{VhSoGK<0fGJ1525N!nt%iu9>D`$8|BY{K#Gjb#a zcEpZGhu+^o2d2@;q=o8_jIb&c4~rwtX^DNfeOawtHSnX2QhZuxt&rdS#_1<4Et>^; z=%-%KG>O#dw6CAR;m3+OH{c!!>A<++_Dg~54f!$XxP@H~khjGI*> zSMu`jkl3$4SKJUpkm}>t@QiOU)TQXqRJK8xtm;JFG`;rPDmO`LHo9-$0zRh;^DF{< zoGJvWEIa_U#vF>0*uYx-09A%?;o*J6SmbSv4vizpMC zbrORwRi6^?eDjD-_!HBgWl!*)b78lOrO#v~Gu_;xhJZnE&w8r5zGqwLLQ$H?pShPs z5AOb;>=}?{_AdV{63JzNkJ)(jt>gy(5m^%CuniBQt9OFI(Dixb38nC&_sIpq5O!Ic z5IL7y=q8LG5G;-FuBbikx^Rn*U5hO6(iVer&tqRf5_=>-JAOf(ZWj1U+X{L4Nd$E9 zBPbl>FU14tmL`~*0Sh*=KFOwOhN|W3s+^9efUB$W0Pp^e1mrv9T_#OB#GB#kV(RC8 zKUwbBeOg(oTbF;Qzd7usToa--?t*^@aw+VMCkq!t+D4cs3U7Dzt0o$_W2-oWI)w80 zMksnIlZ3usk1AFGt0{GIy3c$8c?c2mX+F0>J9HeEkaj1LwrD z)En=(voc@R^F9EC%k8*OVs!P)$ti?4lU@gVOwyR4M~*(40Aospe!2_1W3G>eYqb3G z?9BKQA68Vq>va5Da(u5!ubL_T5q)hThYXaK4B>Q3kgD#rc^<@-yDVk1Un0G zQ0C5GVi-EQh_#p#`;2XSfC=FobK_jc^V(Lu-U(FyJw#o(z=@LQkKZG2_q}s0N$@Mn zFL}A(`MS&cU8%e4L;?C7w5hW5E9ybi0(*}h*&G z^K2Y3N^H4N)NOM-DhiG*)1%+bn$3Z$ZMS+dj;;pK9M0fJ0i2lp4j7L32)<6wppT^q zK;{`f;TRirvh5Qc&3oGgD}Hi?nVgY&`tH(;cWQiSK#jGVP=`4Q!z@0-iJ$nlfpfVI za^#Sg{m7NfG0<3NLgJ1N&~JDdWqLyWS>fDL`(^A-mb*pb3L=HQj*m4TQ*rAI2Pm$c zrgTDO-TPVjl+{&1XC?#r1GR{&vG&!z{UXk=aiyq#?#(e>FJ&kAQ4pDd#w#%v1FrnP zS*_hh))>S06vdec{@@G^v5a@(`Is{};V%xTinn26UB?o3D)0?Lq-}oXoKXig*PdVx z7m1myi`0tQ(cgk@u*s1^xxLa_P=`YPQeX|bmq{q^W5!vJhCtq3foa`^yZ8#ZC_-GAxm(>xr_e$;bLnKB2+DY z%QyUHkbqEia0n7Z3c!7bo}4pSQ})FG129(7gijUtUR{SvF{(MQDRA?#>J>Zpe>sM; zucV`P6h;RBnI?qjK82Q-VveW{ku&=M>;jNYlftA2#u$40;HZxs6?mzbr+xiG0O)AQs>^#+?fL-Y(oe z*#|ipiz<(+y0;whhIrV_p3pSF*ysXi#nb;ce#*nBtr5n(a^!GugbG1?DGhLnZ5mcgw|$)r$%7jB z9&j1sdu02jpjOFC<3F4f{immu{$9V&C(E+Z5Xzuyhy;l~zME7Iaaw5h5W_2<1cJxq zbwOdTK~J87r?oEmBujFz@juhk8LR5J%uMW|jK1PNe!>quT(gY2WeU7JY@e3!x@gJv z>qFm^hW7djWUOAi|Dz*kw86Ijdc7-`{a2Jl#OS|*8;mm2_UG}>nJ5)L?XK`^#&FcN z^UGIdATKr2moT^U7sO*g*ji(g>#IFfW_nTE?@`y|_|0=zDU>)L=W|IeoGK4l&h+H! zNUzy*9f8_G1|E3}$ATz@^YHzPkKE&R$~^==ebLj}(qzG2i^uZKv*5EF;Wqv+P=63ZxRg%)pJ<#oL8!_#U4vd!(h=Qnu3*y0oxEMh z$MlTp-ZR}52t*$iR7P7U-BPVS^mfPCL}2ixWpo&y2pO^AYOV2nVgoN%1!{ldv+%um zy&p9{6Si*5x{))o03R_+i!~~ZK>-M7$}T$i_=aHH4Xdb!Sgl+7pThX zMWa9-p}?Tf_s)fEnVIYltQ4Q&k>krgZ2i45Zd%{d+$Q?_eVK>?=`9A~S-vevs6U}Dy ziN=FOEjXU_=iMM-n-`XckF>cg$$kqrDXC*Q+wcm1Hp{ZZ#WI}Yc*WMHMl8R1QCJ2k z>}|NJUgD*2qpek(ZwoeSo!8`Wyf;04J31Oy6H$;ec>bY(^hk5sITdQcG-KE}3m*OT zRM&b+!$)MG`rTb8PV->?gj;Rv<8>4NIyxmvI!X!CoI^H&oex>Q7DXxKQ}A;Pd^4I# zwpWlniMw^LDoFDVPr#4ZwBZ1#gxHof*n*`kgCwu#HF1hopHN`_IpbJjW@?;h-J4Nf znQ*-*Ni$pBK_ud;J6Wi|s;sdtfd$7jeB|Ryh)#B&Yu04`*a(L>FE$ z;~~Z1xoLc*F+F_JicE$d7vp@AjuCUbjVB0A0^>X*y?ra^!zaeacO#-aym%~caD=uY z;{|!jb}cdFb$l8!VRe_P-tn4j&IR3 zm#?VD@JPaD?zBBGD^U;O#GlgSCt71Xp~!dd%2QAh7>8Qxyqy-oE~Ao7ajdDYCye19 z(R%uN{UuSpKec8(x?M2rbu@(W_4!9txwc2zYwmIfj9y&5QUMvGpWjMyp#Z@ZLw^}( zx1eqXWIjpuNrCg_glppsX109+%0CJ;7R*p4Hmy2zri8A!?5gzD zANLXybu+>`8WXfw>IKW)7nDo>KknW$s>!YG77l_G=^(u-ASxh=Af14PA|e)~3IPFW z(xeK3Pz32!P!TDD6agb0DT(wZy#(om-V>w*NV3mj?|sgB-e-J2&hPIB!!a0`akIGB zRpvFXIn{;IsZp_{NaPl#ZG0)?ir6vz>XHZbc0z|}>@8~ygQimtdQOL^*$bWHtnWL< zgOv7U3rMJBHZ!vkv|uB9Xtv)7yWCW?brc;p5C1zY=|RDC0pJbL{nuup+ z{zb>ZSF3wX<>;%I+IeItHcv>gfdS+}zp$z8HYiKGkGp80^TlwE=N+*(F8vhvaiM;^ zZ42A9!&i4HH)M&6nZLA(huc#a)1N6wW+e1ICcOK)b|jd3l#-g&WaG=Dq5t+&F&#ht z>4gtCVAHde?dmN3Ca)sS=*&>h`4=%h*5N0DS_cC*uPXfM#GVbRu$Et2(|Mp%RB5JH zeIw25OZnC3;i55#{d4L?vJ@@E7ii$9ebzT)rwO>OJ}ZBg)aKTuLkTPm58F0+Ti++? zbc*gF2how!LN$2(Ah{m4F75dr4OR9z4r(hdw0(;(MjhNO?B$LQ6>0AY9ln(L#seX+ zv;G&PG&S3YtL5W5RZW74etxH9!q%b5NI!LZ0Ruwm4>YDWY&r-3xYaelU38l>-{Tb5 zi_KUu+dDw|j#^>{ed1()35Skx_3C3Y{-kue(h0DOPA^Dy{pJ=AuHO5`jO}Zb28Qxn zu=>G@S8@`3$DOXvGN>Ow5`R}EXbZ>8_R}N~G|Aid@M`aT@+WnjJve^Zip73!UZOl? zhviq|^j`z?@o2qe{Dn`NXl=mv4;CO>>LnH^iq0nac_JGMAV?`v-X49!^;uMdTBf^L z{V%JCdgsu}#V1yoZrVhr;i6E&{G8b~FI;A1SMO2|S2^uMjJ6whPSIxHi_La0ApWEp z;D;H4m5PxDF!guxKSl;NRdrr{UEL+*%NMlQ&anu?lF+bjYYmF`_o z?90NA)chca~~lup*D>(ljt4R!lauv6YDMDo=K zx4)le@@ux{X{rAzx~f(R}oC(yhbC-={= z&DDmR$Pu5uktsy!P#mf^9ZZO{pVb10NI$_Q#P+l*G=i%De3_hR8-`Z>!BMnki z+YDmjX5&WYn}VSFa1n4+O&W9Nd?r^9X&#h?Z+A`Ab^j4Dnn-7u_r4ySHu}wJE~R20 zY|=|O`7k6rZO$Fnl6+KX6x(v~%y*R#+D7VHQ%E=7qJ$JY7hr9Cm6YW$>S2DUTy>aX z$~BSBZ^zxs2v_t!zD2y7w&q}k$*I=zwhdS2dzl#G`(!fgyV6UYy$h%z!Ut#7<>q*f z3DaAm2iG0s-CNJzHCgbW5Bf?YngkfUOQFC|p%74Mps^NL@bWDe>!H}_f=XqlmGt&= z;R+U0IhyczDRoPA;m{2YZefxM>@>M^f2aqK)Jf&GNm4jh^UB;Hk@vm1hV__jQ0JEV zHq7szj_PS2kS^Ap6FTLjv{_mERNt0*H}2-mR8fj_kC^o1VDS^6{rdHTu|1p9De|1+T!D^2NWdOvo~;aOQ4R;aSu+gj)8x21>NMvt=GbQDDK zF_z0s=(#MiIHJD`iaP}`mspI)vGc?|6{jq7ZX3?c{P&}e@~x~ylCpL?7?@OB;ee)< z3!_yrai2%LQId|3cE(o!sEchWZPC$N717l06K4C+uFxyvp0@F~Hocj{rWy_5neuy^ z@SDjU0KLCk*%DI{I#x&}I2(E*PJcr1fVUs(5`h5hYZWWkEP@4ARrP$e$zlzwDDo0~ z&TZ{xaNK129x`77AQ}^#)4pWM+?`wJHRN7fJdMefNH9}S1^i`oZg=2($S(3hp#FB* zQwHyHy2@bcKUq8vE=0Nm({sRngn$*Ze{)(y_4+Ed*@n%uFkA-S)C7)C*w^a zd(7kP^RCfERXpz>bzIM+pKV@7Vo^JD8-8fmp>Rg%nN=aFHmm z_Oc7n<>s)3?BsquMX{Qokrz1ZJ=BP=gC*J9#3_zWYkwFHJ|7`s-sKlSvJWl7b!Pq;?W$H@n?uCxUKBDv78DthQo#(C3d5`^l4D>)t0R`?Y8S} z+9J!Y1bb-Y%tlyaz$q1H~#bc+<%lnv|{I zoSJuTQjZv(el^u8yi0VNc3Rk;2%q*S@HJexb*{=dKc>54gW)dAkJa>3?wjg@0JCMJ z!2_>d7%}1R-()v69`Et^-M&vPet7@J;M!=>0(Up-*`!xRED6EAGL2CoN|vnY@nP$0 z0qplt&{B&MmhvAm%rS1=T{n zym)n`9&Tno7{8DqKd@QZqpA8>zqUr#L27!O8+Ckol81#w(__-#D1opx?hh8VzE3qv zvPwHgDlQ0RIHf`7Ybn z(7xD0_en$AMoUs%xR1t3Xk&bxJiOr9IEB5n|C*3a3&kacZtl~&lI6{PHcOn9h&qE; z(s=HauU=xi_h7@$EgR(}k*c|Mm@3HR!2IUNS5tDIq1uhI@~St6XnF{b6u%6o zjSR$ZHv61@L8rnp81tRHi0ECJcp?OlE9bq|1Z9ShkN4$pL}$B~U`N`cBk#g#5t)u# ziE(GcA7`Y;n;rt;nHkbMHE|~t^!sRu%DsWfsntAjv(`H?9c8YECI+YH?HjG6eig2x(J>=Jwe^e{QNx5*uS0WUr!ex4X5rV7@$t|Z8rFRcoq+Uj`_c6$yGJsGBD9j!iHKy## zZ|FJ8S}07`!O`hfWrg2Ua@52A3JU_sN@<4Qs~darySf z6O&qo#EHzmAWz_W;+u5%Yq`&1tY1%gR2Z9C`ZKg=_M#dItFCf0P0+KA&J4-F_Vub1 zhRgX`d@0A#iUq)a;&=Q!O_Yi)wtI@ERi4!mG9fo?bY3)ae%yeg=^~ZD@72Vpqi3iK zq!^GcqkK*_`KU}^L<2C{KytN$y`Q~e9z!#T`uz`*QIi=%zEX>mZ;myy7@Cc8syQX9 zd+G3&@@obyxJ$1xw$5=Uemrj62^nI#V|Gilw}dLp`ccc}Utu5PexB0VI1Pc9{(Ya7 zGYN%SWQZ98UweT5?SVx`z`fjy9Re?=N_1|Dm_>YRabR~Aceq(_DshsZ5~H|@_X?0Y zW}N^$?!AEY`8Oc`AD!cw|Hf$p3Sk`NarA%a_yD%U z`UDz`n-3))0$3c3D1e=4EN}rrfHF48*k&f%?FO=0o7 z(YjL>$B&2hfEfPskR9)L+ut8_87I+-3aYMe9TEEoU~uxEvU7Prig-DGlNEe7|3pC% zeI(_ytAtiT*vB7Asa*yGn-nn8a01vg%z)f1qU zPtb$fIq`BPmK?$+l%9lVFD!Q-o7d`=EA+}!#?0ESNT>nLP|nJlD1R8^Ng)_ z^M;x6>#2c*}00Clf~d#MZMz?jM6_G;I0wlk4ZQyZHvo{aB_^W2!@au)}z zS6?KSN1tl;s_3p#iWT%rW>P05%_tS;Fh2);cqzvf7j2gJr7+^Y`#Cw<; zq&80w!MipFUc@uDMQ0JbrI}%U+=-kD)*4>luL-}UOyLnxjU?I;l8-f31kd1Q$AyQ! z?MZwLyu!$)GKsi!_7!f{xJz=rEiM$fqtg0G-#{V!31svc=$yDVdG53 zsWtEY10a=c>nalFAD;MBlldhuyy(X(b@S@*FvL+_5@;1ZOZ5Z32@<3THGS zt(R1VSP1w(e%YQt6r+#UH~K!_F1%Q-f%BGF7?v}pY*U1)=#oH$rN)tA>no#;A3$zW zq8Yls2gDg4VZQeNEm3J9hi}Ked#xf~V6v)quKyYS^wx}WM-um>dVbJvH_iLXenw@O zW*Erx#8f)ydJ{7#El63yH6F@1xOPE^(fQov*~21d*SE{t4AKurQCUN-vfJzgKfVGg zpEQeXh-F!xq4Aift*Qo1g3}HWv$FgX+-*?{Hqfwo!n%TF0qKMi7f(7w7mT*rqt3X0%01}b2TcjnfQb4e3gRN7b0gxpOE zdP%LXwuhJUv)dQ}R}Kx<%l1ZLBvv3sEP^k>E{DM7ur>o;eZaXx;O6*Mn;9w>sp_D{ zm$N&XbnVr1EvrI;DBk9;zdO*(&Ko9TkFLyDR8KKO;m%JNyynw((BuAO-OR|pAk)?V zXw?A)MtST^i!_6;QDf@C_-2{osNZ{7xyk$Hl8!?&1*;gyEssqLFqfsDnM0D;>7hhw zaIvrsP)FB8F*E~ueGIV@QM7ayLJ}c7oM&Wjn`WvShAToUDPO>X%wV=`PW3SNn&^Yi?YtMC+~Gb$qfTRW|8tNcnxIY;0pqeN z5KKb|;B~|ddVW&Yta1tv(XaSkZ6_mZ_`Ue9agaPq7Q-w>rj?O>7DIr3)3nftOWL{t zSk#zEh48q%m{05wpf+aLd^Zn&-LetX07n-b6QA`!_;56?~0cAg7rS49YcrwQA}&5$F*-jOLm3gd`8c0zPM z9z<`@bCKgSP>7QY!n`(a7Y&zlE_7IQsVKS0W}uxelU^ioiKxNxn9QP23shW7vOxSt z36U@Si2q$j){Y0B=P{W_itB>~r@np2H*qkGcFHrEFc~itv#O4?R@WCllXW6q@;5oh zhy(A@mL?RUA|iJXDHXO^vAH1jK^<&kKBF7cnCGVYvua)fUq5z6bOZzG?|X=q{4Azh zWG*xiX}rLRAkc#g-h!zkN)W>4GT8~70Y;j^vL#SGqhmc}>RHpU)A47n#n|9spDK5k zH^5X{R30!c{6Rp-?Nb-fo7ptWy5lA5=8Jfk#_^I3CdcI$+H75=rSJ_sXVhLogQbax zixfy1{fIcvRc-QKA=6&+Cu@Hg&vF6HP-dva2ms*@uVnBYJq04svK17&)h+088V#IQ z4Oqp~V zet3Kv(dmFUBEf++;@|pf5_ltQYBX+Lqik4u)a-|AoRn1d+!xBzAAzTu)nU|5Cc#OH zogDT_mkeT)2dqk-7^KFJz*P2;gPmbA{WL4f;bd`Ne{;es=@^j@b3@Jpv4cuUIHe#H zS^<3#?)5~A)AH@{;U%)KiLw>RlK1~OyQUsMrLNDtA*OMG$?gd|g_`#;E=Ube;KFM) z30|#34c04W=Y{%4?}xrf_)PcKlB)|MBSWJ$vf+j2+He3N!DXJ@=z&;q$4pfG{Re)e z9ew_+7elUPhzz6DS&ANjCeh6aBCx6bul|A*z@bFF7Y?QtnYm^GAMch{mR3u`x~d|r z&D#r`g0BB8diy`8+9y-)|1N%zl@eI=gc12}D#-Mf_UK*sF2x_xe-%GSR-_9l9;Dn- zEua#~{ugAUg5p3<3ur|M4*o4WsPhnHxAqG3mJsm1NZI`lnFh~HM#RErf)Tl?3Vo#L zwA+NfP6m{_&O12dza9juVGxeP`Tz6hLjsLV5)-mdXk3J>s~?44UqtMh_S2+e{3}nGF&-CcO3bLF zT>~2Zd$Cz;LVAmW>e_>>ml|#NeOHG^U=f++AXWiIQN_xl=#M0|E%)O&$L=akGM*tn zKota&Z%B@bsCYV~vTzeeSMcc7Q72g&lH-nW9v3?0WCYgW@98*4%7FpHwND!T~ zj_bS;_I1qhq2`3Z#ul5cVVm3CEHRAA<}S0>BKQNUr(L!TAIS!q^>dTGh zc&YwEN)C4Q?kIEGJ<}0Q?i4-9Vji60Y9;ewT*fhXFuSpuiw`=iDjMSK&d{E&Dq@dG z?5ja45$167KP|)?N=H;+C0beLDOXE6YonM|wUqTgG`9JB^uPmfFCCo8lD?{&3%-+U zRRgJ-m+w|Y6)S8QBxt#5gIV+UOiK-Uw6^0 zb04CE&hviBgJstJ>{C@h7E}Z%4QUUS$Zjfeg!GPzMP0;vF5o_emRjEpovqlJB#Ps5 zF>Li^s7zKnZ^`z>Ej@^Lioh?{$%$^Nm9krx-+EdlbZOE;!*uRI;G_a_w;D?wIs?o2 z*tjxIOY*zsKuU~Y^X2<$dA0iCnH2ErvryJ6l@1;O2!CFV*HbcBHck4A4-|#x)Aq(M zQZ@(55w8(mA$^ksV6B1Y;+w}~fB74K?65l4egJu)GaMY8IKlHxM4NC87pQ00a}K|2 z`a1SW54tObKOiSpYZBoa={M{hjPZunMsA=UE(YUUYsGBui z_ZP6O06E*JnsC`f9On`$yQ7lSY2~uD_43y}M;*~UxpH;>_GTp2su~4u*z`tt+&KoD zR3I|wKntl8dI|wd9DPta7+$Ex&M?`_>pGXLA8CGi*&)g@HQi31A{sPd65OV-JT^KN z=uUL4Nt?&cMC-L6MXr32Iklm4rub%@KLkPvCPha>DjsuskB%gyuD6VfEiU9n%vPhL zbohgl!wf}nbVP~I$gX3D#VXM7IoO2zzE^pv*=oYgA40pp;*VJi`yWa^ucX^yZ-8{X z@z9sCQ)(|!4YH2Pv^$pTU42HQuKlEQJ9rhQ_JC%M>y4}vGqb(7pJDu|#{BU%k#IT1 zpX=!dAR2G!*<;ely+b|jI9Su*Q|%q}?lZ9?!|`=DHFf>w>r*#Ul) zE6_-fHaG3%QM8=XZuOiexB#-aw-}lF^3GgSl_c%f4`WTQfv0)$bdZVOfhSW>AXRYEYTT8~Qybrja*G$qXiqI8GvP|iEvihI`y)xlj31wmVgY|qcvdDRCYmUpZ60mz z)WWNo>1m)4zO}Cab6D{%r;KA(8`)vra8M~F8q|e}x-+EN6t8e^J`=dAy``zYEIyQQgn%f-)460 zfSiP?=Ge7cQn~AwsRc9*5;N`uni~zyl%56UUXwmnTcv$RFWtp@YA1CjW*$TJXB)^V z0po^$7Y6?AOYHFP!2weC)bMu+})|g>A#^k zNI=j3tu7xK_(1WS(#LNOAJ8lYoQ%W%{Spvwh8;;%fi0`|`TJvK()D*DsD9|6YGLs;a|A&kUXL9q<<`2H_jac`$G;PEj-z&0;Q z620F52{8IcDyTxfk@>7IvVD*c+AQJvig80*EOOB}jrz6$S>(?t>X)i*O6*IiGvz)l zIxgqq%;+zQ_?#K6#Igis7zgPntpDus0t+pDtIs$7Pd_e#Gb9y~^|<<#Yjx=J8VV=ujH^~-rbsi1QKxHBmTrKD3ZJmh{D#;NOL~>PK8r8*#VtSPodYJKEgs6hkgUU<~ny=Zgr(tEY&~arRYQMh_J_F?C1IdX;A8F z;on@b(D0d7GWDn@_czXQ-4zBWi7T`D!ZGppq6nfyGdytmVXQ?1*1X*AOmLw1`T?{& z9Epvtjta28H{;6R`(dWq(#So-Y?CBAw_XATaU3_0eiYp4Y?Dj zbi(zqP_8v4L3E*R!T84;by_vbxv?n&Zp_)tLlf)36rpA2_k7#C45d-m-HRoz8`9rw z>U0>^Ss#mVUVIY&;QsYV>s13d&#{8Wx z=4oOlu$&F2;~ISO$4#bn9T+&{9-h+N$T|kpRueKEp=Z7h%CY)VA2~hKgrVb4J-BZn zZ>X#-0(3I{NO>q$r)kyU-4a-{Cf#!N>+x(y=D$3%|8ac-1JF6QU)z1}!sck!JP%7T}?a=Qv)YbK-q^M|QG{ zp6$%mE*2G^#T>78Hn1=7+EAddeQgFRwThG1>z7aQx=hy~_vF=>%NeqmI6q zin{|Hi^&5h_3IPjC|bx~t;C;?WX^5SIj^_?z;F8yS%9$V6IjoW1oJw(FvhXRc9AeJ3#(O{eB0m$Qj4@cZwIOsA_OkgkSY45OI)7MHZV ztX8zJN>N8N`;~-HtPmX|vmS19eT;r2$-(YTq}t7_i!RL>_LM8&RNs*~ms^;R_$m>{ zWT#E&?}Jv;cQJw7t4DcNonp9G!ml{o(bksN4pdht2TpiD|8hJ&5R|dZ0)#jBFe);Z zHAxZ=ZRLvN&R^!3Y7Q5tQSK0_qIsI`TTQ(F2FTu@9!Jc+fX!QE9-9z#@X+x3CtvI> z1cz;goO_a{JEeY1h{g?Sf7C^IQuV6pEBJBMi z?IHoGNr=7h=Cfn@YO3Lbw&so8oV&fjksdQVKLyb2>ip*DjVU~v4pv9@Si$=FnL)Xe zgroyWZm*Wx?|Z!)zrjqua-{ISSN8bgi8GChk2HOnV-{M8 zjIZ3Lao}Zq_r|3sdd>B^o`(=yL3DRh;RaoD29aVt6YSB%)HQO7!>PE4lS1yk3zB9g9&M)??{5Ufe34LN;KX zfz4Nc-v!_78?UcaZ)e9c!A!B12jk>yiyVJ{^O8pX)DC{DhWjQ%)_v8fHj}I z)$7OCh+>#}3a=u^KvDa18C!DIaicfbShBXJjrskbp{xO5^aD=`-MO)L616e?GiCZ9 z924@6%?9glAuePK)a!bj9r`p_C{^)od+VST5+HsjFwhq{uOBbR%(BMCLI~e)6aP?U zHvU6<{^8FvYN2?Q(HZc*#|Yj4pPf8+3{q|F@QdwjtFkou>MM1&i4VT-TBV0>g6-mC zz(f3J(tPqSN`h30SSWOW?NdhsY3e?hqtX8?EmtU)kM_66P7KM;BKGE>+xsLkB341;}T z0CT#4C+IUO@6pRih?x`JmKEd(^JVDaLgqn7PFFnnl{jhVBIOP$iu7P7=PyWx&R-CC zlnQu%|NVIt)M>)wCxB7s%vdquU^?ElBT*C``=&e6+HzTHNSvDzDR(5IT~KsjAl?56 zh!kW%E3}BInH&gA7f2eF$o@4V{%>ayLJ=8d1ZJHjkXJqbS&hO={&O;`F7)K@HR9?` z*F1~gL<>~IkE&E<HU&r~$b5n7R zH62)#9&+R+1-D3)U5d~F*G?8QgcRdhw=NA?t2RmIqkYExm~q?5 zfp?WQAA3#Cg#0S@j?JeODNs;V=&LD{;4dpw%LNSt>RtZST5VWE_$ODfp2g+}Z1}ZQ zZ`H5>onzZot{q>G#&-kb9iyodvhp$o9g*p`zRTx%U+udWKWe{8ws0*QmxfVa=Ug}4{1a_IGjq%EX4&f8l-iscPh0JgIM>NN{#x6WJ6wC@WK~W7!S0cDq_WjH& zs^z72Sf+!V{1t{UbA2ZDRcT7eLQUFo=Y>0>3iPElRMx6% znScl3D50m-VtB(eGyHUHNlVCun(*~r`_5T`FVvdu83-lFk$2RNejJo%GmyK_%2qIK zJoSqw%IEFiEA2#^*op5iUn*IT*5{LX$Ct^k2?*RX1Yru|cd#kYI3-^Psr8i^WzrT=fs}Ml$9Dj-nA7`#hV4* zL;UH1ZYQb>T?%aRGhc)G#vAxMaOo#;&+2qEm4}NK?EO5CDm@#O9k@cda_Pjh77vTf z6mUlIJEb0}eoGqv^yl`4_bP2?WA&#R$6wxOg!n)4b=`dP1?GsLn@!fQOr3k-#l5wq zQku+v^OC_2!JyjZe*RtR8U?hILrlR1-0NpDn%L@v1m)RRaPl+MN8EI&0~X3`?6}XD z)_Zh1?dp1HWw;yQ<-;%-z*N7H65ud5!|KSzS$4&%MVf)xUG&5Q^q+IWNy#-kfF6@OwamdT7Cl z58V%~!f{25^-Ib=E}Swu|M6Xt$C(_N@F%dhTZhJM0p!XyNC^&p9>& zU`>A}GnoO@{(L}O@FG$H$J`nq(F$X9g~LSIO>gB#22N4SzfY58YShOw=Cb&%kJmqU z9UpyoJwin|mc{ye!y$u_iVbZqlPa)MF@t?V2=3NRNn>Xs!$`$oMD5U!x+XAxO8Zl% z(c?*>CE7K{fQLA<`JU<)7)#IRsg=zT*i-S*D&s@oUWe46S?!|DEVcUnl-xjESXe`_n8vi=z{A^Ga*3@Zjxn1S- z{t6rusgFNBc0F!hvCTarWl!Yt_hv{S(4muX+D88N(M1ggGCL%!84jB1NtX2%3OAqi z?n?KM+Jjwy9xj0#Q|tol@tHu&bBB^fT(0f#7@8R8`3n>?tG;T#6`ziW`4T+|Lv}dKgYSbwI!fQZ zxO0u0$|rYeW5wPIOx)M8b-C8hzu%XNXf^^J@lCyIGEF8~PF-*t$91n^fXwRa(r!?d zG`gd}T`$8Rvz7SX##=!oYP*Q5F+-#b?;5td>7&a=uRBI;G2vpU5oHN{%}OvF`G-7! z!)*HGS&YL?f1qlrl9ZLl&foaMy3<+8VNAJ3yWq9DCb4Czc?VISjmnXwIJBaYnbhaM zfv+#uZe){v^y}As2*gWtA33=F=7ziy*z0F7o@oK1E34k$_4%wXuCs+dj=xD;f!CEwmhIKzy_{-?Ms#neOCMMU%ec)yXh1S`DL zJ$!YDoDfU5v(%gYXLS}$`X3g<9vvF~oV*dU+XL}?!m(2_vuVHTb+r-~)llin&Wnhrl>1~UbKB*)DMuX&qMy)GS;j=gcSQeN zqqIS7^sXH15w!UpRHjav&**s3yuWG_nRN9VP4>9^n&QeGe{);k{HNu(_BV2jb`O~M zmAPZeXfcxN8#E>GZt+cL1#eb@+W8N&pVUL+j}K0a1T3!OwB}Q)TS}g7>(mUI-5Wk1 z(Z^dn{Zcz9Xy(R12pzw5!%NV=A{=T|g3o#CIDzp5^?K0EMn^SG&0VSkoZHV&pd)9175n7@P%DNFO~&c{Jm( z-)5$1$)mQt(`b)$uYKTyy>RC*h)Oc~kRudN)9P(Ohl?dQ6yzqEjTew6t(Ux?pZTgP zqs;?Um_Ll{f~mJo9_$JdEAX)73$Opgm)c3z*AA4Y3)c9So@5KX}<_yUpX-`)+N7KMUXR*o!_V2#A z+!D!&1QrdNY#OsXdWR)b@P2b2Z>cDg6s?tf49_x8WQjgB1w)cOc8R2WYfDnV)pq&+EdE31mo z>UiZsoQPcRfL24$^a8`hJrzOKqSiHkiMnXM8kgdwB>D(rFv1zL6{!Kvb|3G+FF73R z5H+eR!YMuSe_5-C>o{xp{A@E`PIA;IIAlgZ|6p_hJyxF#(9&O*oaW;H=Qo9LLr@mP z^0#+Di6fKb2RY7&B3P4o?Xb+mJ%BLdz>o*2MU?Gp5jtE4Jj+>t3|j=LEq_LF14#D+ zA@nbZswbLA1vnN1;M}j{X~=%72m&3T?g5(S3Njozmlr4ugn`Ya0e200w2NrIN7Mk- zc`b~53bD6Bx#dS(Cy+c&&$L0$_BhFU+;*vY6B~Nz+*#Jd6%C;LxVxfW;{dOAD+#^8f631daCOXc*j*0#0J9Ch`D~*H>-tzaRv7 zc*!tf#`JaN3VbCBS-ihW9tv*mYbOAd}$tMKI|{K z;PjiloBe6$lC0CDIKLfwXx%hWijXqLOuHswg_U|a-GxukS>*7s%+FTV0LyCv0UXQY z`faXnj;eB8t*xR_+QdELWs-A0dz2W zM;k5%=s_#!UEcZgq_jYx_RET)j{_vQD(&#<7KL z3=oJQ0rZr@URjKOop_&re11Vq_<6hrNwCOdM6Wo2S`N()xN~C!*A`ys=v|Ehd!GbH zCC@LPC39lp&Xgr41VpT8tN^(bZZD)BDGhLtstIP~+8*#JHb@Hqhe7RHMz>S9pC*mZ zJ+~4y9PF~z?7Lj|Ack^9&pVUKllisbv{=USPo3>p6oS>s0IMw zK>CC;nIHLoIg=7w`f+MP1U)Y-pHrsL3?tMiRgkgn>Z=F*)NWs=t_9s!V(nF!1;Q|a zeXhl0C8!3emL(fH#+I^(sj{oZ_be?BI?qz*@8^L(q7kl-+|W*rh(cK-2mAPa+12si zg3>g9*pxW2QnPci&WZ@R#n=68Q+igvOyE0ZP5na8+?@x}*nuy_0Is6Ef?y*YsVKjCzfxxk(wPH?aCFlbVea zm(G^GeVh?>+lF!L9GaJ*v^p>b{|l+CyI716FB7HXC%_X=C-!&jcJ{MwKr z0vsj{N}Hy&@+cp6jUdzvr4Ylrq^7AAyH>JaJyxxp=y;Z@KSw14d9^~fFzb@v@Rk9FDy)4lAt zKV&k9J8dhJ6VT_|hv(HefoPTth(D_mgKOcT4*4ebRfb3B&D!%sCHAdcLxtYOgaG~G z^^C(F*vW`3XjJVQfl_OD%ae4grK)t=Ofk;Dcp;!zDduZpB5?5_N6puZI07dXdkRi=vc%PG{IUa&_a`j zZvQ|Nru+eT`H8PFb(&Ti&LZ`#LOHf^;dtq#uhj*0(fJtG!62w&tY7s8fHTF0ni0C0 zqzkDHx1QkIH+E~kV0(~#3?X)m0`+AT_CEZt&(U61X3=i<00I5y6y==iw+4R?+!4W% zx56#9ZSHq&xQyFfnkTdAr!@VUu6NK>|2v_kSO)Yp^g=A5oLnFOhvN_fEA&f={g!V9 zOslDUIFqyEjB|Mm9}lxyD2i_k;0~ugb`E1iKNb1>+~pKZpZUkfyZ9 zDe?UpcWI4+%h(!7p0>|WNhexWE_iX0Z$3GO!->p_{5?SqSMEDL1RunQv?uW7!c~WM zKOP41O=`X3>n>ZGdtZ??%aZR#CGFnEtm>{Vn0dlK{m(C$IC(;l91u{?)JTgKhEZJ0h1|_ zKu0RNA1J>bIAF$13`$v$+lRIzf_SUr;Ev>TxTQjC_9@LGuhYj$R5nkZTztqU9NZyn z8JDjBQ>`MsB|gJklNv3ZBW!jDi?mDjcKuigy9T+hY-2YJDuwuQF0$VI9N?LcBUXix z?~ArGqlIX-qES5d(>#M-l&9OG62;QrsT606uwrlKh)DFR1Mmnq+!(d}1=&3d2>Dc? z!~kZ71L$Y>;T%|gfJJEl3NdD2WGRy(khdE*)Z}4oG2QNi@R1+C!XSL8JvPok2T2T4 z6b?A;XAa276L{kCymU15pA{YS?V=^27-)H3H8xWKwvG}QkzEWY9+tJjw_=h!&)pTV z_1z8yYsg7QLbujo1xNbCkPXn8oUG`eG(dl13q7m$<%L|V-JNLtD=PbyNk(tQ1F~}{ zp^p!Dk)={|#==o&MRV<0*fEf2L1J>tnHA#1#CR|PC;(4B+Q9T3?lad>Z$8Ipq>uWRv5ogG;h9erD$%_`@*Ym zEz%Lvgp`4zb%Zf_O>8<@3>R-niv^80GbTQdIcNL)`ST~rWmypFYZ*UgDHqU@nLCeE zjCv7RXRkiwc3K%xzCEI%9?|!Aot#S+A^#!^kHQ^_95Vz)D$w6&?jM-B<|c}2c}sTh zX#R}JV5Z@>j!Cm%CcRV9a>AOcEyG53-ws)(*uL9%q`gx$lyFwo@})YDc>FDOA@d`z zFS!erkNRoi$s-_V-`+zUZGxnqV~CTue<+!#Tnzrf&hl1i$xs$0h6Lza(kP;)f=0*z=OH0;cdB3H^ zHqdjuvp4yyJ-h&h0Fw9pt=BDv{vi?h(rgxJ6U@HM=kNbAIVnpqXiO&jB>_Ie24@;q zZNQFOG%8I^D95x7UM>rRpYl%F>{#IE=eGD%H(#fO`E1M zH@OpekMvQ2Ff|wG*DXzsNUCel-8aBLo0+`wvDp6QZyGh-N!`oZz?kzP4}?VbZD9l{d{ITgQ$NV)Fm5+FwjL#Zv1`eBX^zm==zjA?1{Y{t0kJB0K)t*2v zNj>^!{P|<{F<^#269jDbIGcXCen7wB*Ybv%aJ{{cWW7#hKJHw&z3u4CEAJ$do?i5q zNnrf(ZC&}J^>1#k`_X>9YT4$8XQO?@X0PHNY{77oH&|_BZ{HtX&<|TON7w@n!R857#`M`t0_L#>L!|FROf< z^WZW24d7nN7W4VQ4cn*Z%Kf`{J7~k5ZrO`Unt~nbs*Vahe8(E>gwEF9{H^l&L)u|= z`;Ql080(_#|Dlc(LX z-#u^Uvj%G`=$=%*%ng8trrCQ+C8}V-*6o2qP^b^=eHL8E{ ze7)zo>W4trC4ES5F4@n2`;KR+Y>%tW^MIu%la4$nkyK}%!}okG@Xij|>wl_!|2_N{ z`ZK{(Wm1uE-HH=yxe>~ z&WeFw|A$E7$L2$Ezgm*NuABdDqven5t3LTDY?uGZnSJ@6;J(*7^O22r4%eJ(^kP^Q zu*j^4UE#~|;5iKZy}P7-)qiXJ&oDXnN9uoum^J&q^juTj4@`ApJ33-s+sXGa9q{;g z+IZ8tN=2Up;l7)aUq#>D7qt`Q7pu{|WTU(E6&rW2sG?ZWoxeA0cKq3(qGe>X_rQa1 zz%fIn@XPx*1J5gX-+BM*d;5QrO@Vur4r~2*{I6-AG_b`A>~sD&TYdZ>u(P0ka6aGH zxy<2LSdU)YvPt*$t-mLl6@AD#;4!H!cDIAOL4SebOce0aXF7`OrJ()H-_ z-&z$1^_E5ZCLjJ3&7LTHQsu;(oews3%+NdrJR13{ZT7w&;=k=*{p0+2#7^m>-10=_ z$YYPLakG{4z3b_T*`>!2X2EQ!zTy}M`y~^k9Xp`&q(nd7f4i;lF=#{H4I5xWKem3O z_L2P=^P9`>3oXBF%F7?HZTpp6fdd!siABsZ@=g&AQ*5-Gq44dvf;`WY>sN&2e;oYs z{9m*nFj$}Md=F`2g~OU+>+AAc4CXf`KYagf&g_=z)@KL*?LPatc>C|!KaR(gpEqIr zz+0>OTW~UP4ywH4SjBIy`WFWy+rNc<-0`1b#`1Ri4@&j_jN>laV zE-v=sV*j}6=vNup)mizfv#auxmHlVw&lUhhT3kvT00ssCfPEZ*KRe)pQX(SyUzHTZ zrDVnab^)9j@KL}~003J%XONPFFtLWF7V%%azZXMeCx^e&|Dyho`>^nrbO2zM;a^<; zzhl9gm^vAMG`jjYok1U+e<*A6L1UQz8}0rVZS-$+^j|bcMM>nNP2LAhZT_EVqyI#^ zIXgIiv>E=Zy|Dx6FM92RR@;n;Jya{AmabQGfD>lnnM5poRxp@$nyXIU} zQ4;__{{;ZRY5!f(f86_zXYqgWU%5Z-0e<{X`;WsMU;{7(5CfzDb^v1l;|IkGUGh>;o%YC0r;OFAfdp(p|9|!W;Shj|eC5}-pGN{-oXfo$*cusw;1Hl1Y0AvV#T4lORjF08GI>xPhHup zt$_A)rmN{wHvxDBaV*}&j7yu|;<^eYWo0UuvhuW#$_jyQq-@%^dFXQlRp&QGM%col zTRl~3p|Sl`+7>U?c(g7<#O$j3rY0RYpZ;0Y(3H{1*m*MP+W*GS=T~DHcyrRIw=u0kK=MFtMN84r| zt5f3%*fkrUuGXkv&P`n6n#V!I2rWYc`P7261WoSdOR`%@uS2z>MEP9N!d^KEz0bC` zT0b2)#4@S#v6l*a7S|kyz=nVhKu|TI#M{onWMTj5JO3gK2J?aDWa((30B1T$RQ<1` zIcc~@?wKHlXastM0J6{=Li3ZVYTD&*50z_lE>&19>~>{LT}0Tuh$D%aq5aO%17!_WkCa-I4d`A-Zq!a<_{jRns&XR5&F3=iKQIxY|LbX zv$MuGXyE8?IKM7C85LdEF3gRPOzJlQ=Uf($ZORx<&IcaW{UD_Dc(TIu2z6c#Q@Ggl z9o{E)qtO}rpE4;$YEda4TX8_}%rMsEQlCT%5?#e)CQSM^*kClu%|D6K58!Vim1kzQ zOU=Oi*cE?=h~+`go_A$*DzAhc<`qH>X3Lp8CXO|>0OU{72D{q~I$Dy8O74{wdfE*` zd{R8^aHffzSVvb|II9HBQ>238Wr%>Izc=F^4&QH11h&j{pVi-U;#T*6dOLX4jPw88 zT@Q)1p@$;_j%yt$K|WTr5pp==VX{Q-o0?-p=dZ9zrstE$OQZy(nP@fAK1E=SMR{-B z0cp<*9O{w(J|PEg5+f}X^UEC9Ahbjhf2T}0r#ESGPb2)9@?v#$l;E;kY#K*ir^rCS zLhM*>$TBpM9%1qnGa24uErE>R)Tl}$`3K-Du$ucaR`LfBknVP@|LiGXA5ex3^uHv0 zUu7k-_598~%o7U(PLx*DDJ(%CAJJE-nZ;^UD|jp=N)Q+9c@=^3Wr1VHMl!}(>@C?f zMBV0GbiT4lA?|P1<~X|ILFguz@*L5h@yG<|CY|w~uZO;&AOns2ulHDxqcMe64_=t5 zCq~DT1AqA6rQJ(IQiIc^5b{|&Po#Kmn=?Sg`C`fJrt7_2zpwoPZ0Jh7B8+uO{{hTS zzj*%?Px_s2&g&eL)+RKf;n>lFtTiZ4+CqtlMP>2$(VAEgvM)>JRa2t5%?%hMWuFP< zz9(}Z&JS5;y2RnCh=J4YjP-dp-MHhH?7YxCcHS5`*Cd+)YK>kl4>z5{v{<;eZ)r2t zeJ^8pvlK}P}5gudbny|KY06vGI}*)kdwv+E31v9b%{o&R_(x!cS)4hgJ} z#GO~KuQEZn5}we6HT-ux*2u{u|= z^kL)W(2LU_fQJjw6&vFl-5-F0#Q7D6@NSCcb=Rf-=ZD@ufG<08Y5qOIru3hIl!Ex& zgCZLeMrPk{NK=Pq05^;(J-;WA;K^eyN~F-r*?lJ_Y!us6Gnd6|<^Ih90O9ejo*gL8 zBZVa|W%)qj^N*t2BoOp0^)!TdokfV7CE>Jhd|k0*G+S5OLxhM85B#P#{CG&ehR@LVeDv0-ZJ#%JU1x6g`FkVVdBouKuO}z@ zSqo1yb9~AYi%`i(8R0!C6TNQ7nSL4m=wXriq5a+c8IVT!;vc|kG3)v$TqiM)zh-W% z>1pEY*b6&@z&EskuZU)0#^#COGQFfX6)u#iQo`a@j(#n&^1~~8#4$1{a+T=ihcUr* z!uuDUbBYgbww+G`$M~c8j&906*Rch*p}c+jyDjT%wT~)kXyP)$uzeH>kJHB~`UG}n z>ePZ)tilDCVbnoZHSPoF(N#^Jt;<7^yS!JXNian;NQDof(is|9gb}GK*zYpYTmHq%M4Z6KBCHpa$#_Jot z*FCo2juWYwGB>e;A??@ZEzPLoGWv3_Kf`4pFp3t5u^Es;hG}gVur5YWS=`Ud&5!FU z(iXT|@eI+auDf}l$AF!lsx_(W*TyGmvN?b1kJ&udo z*21{~tTYnJ1<~@C7oYGxG~nlVh<5z<=LA)v1Nxb*?Uq2Y`kwEqP@^*&u~M6R93R~u zs}SB-B^2Jp6cj;6mq|CBCupUUsvyKtde;`j%^8oNlC z#d^b!G7lgoNFm)LmiX4BEYZ8^@62 zUrq~*%N0DkGRms;!2~6B*g9L>yD1Sj0vDI;E>xI{!)C4B--m84{PfHmE5^v!oI$u4 z5xZfEp~blRxOgfg;6q3)sfePU^R(0P50#rd>I>M7ZKlGLYff~Rf15c+O5WoX&e#~A ziL{?j7oYv?XZ_?e+TiJy4}y2zN)v_g1t}>aMu_YEgR)hBNwC4t&Glz-5k8Er?U%4I zT1$-M@OMbqFep8g(RY+;)CJJAJ5CC1+71q_d@5<_vc0i(2^VL+lir01v}(lXOOYBC zFeeJ>TWam#P_%m1W~9gBo0yHVm~RV|s@NBuuDfSIiWp9(1Mc8hu0^GqUsdSG<&U3} zP-gE*eWh06HR=iT;1W4V^e zDY@Tc;Fs*npwvp2`!p^7XbDgeHf9`}ad5K0D3X9l7ndZ~<6;O0QsE^HjW)I|b5bH) znaf5-5N}+b#C!Ivx>2uR6;(DqdX%U~E4ZWE4dx|I$SsFirmPV71VpDI4(m<*y5ul& z9ncui8<}L2!XsZXg=bWwr`36nB;OEm{z2m!u5~PY)BtaK{ZBZ~i9~@W|HrUfZYv^D zyytxhCEgr0ow>Z4E*KfsJOZJu;%=haONGgZ2&D|BV!fBL_zmP{d+X>=*VeSt#qj{| zf=+&)gZBgb@mqa&6o?^U4OQ%+l7sJ}>w+P0d~ne)z=56H=9CHmW8kmpJ+Fz%1(ttZ zQ=QJDR3Tn0&nWBtL5bnK(8G^+x=ih zpF63zx^cVW58$V@z|8yM<(udAZNz8K7d^bGm(cn*rVHOcfW>MEtPP%qw0|MQdq+N} zb1}63{Vx$RFS%X!r`-e1v9=WdLI}}+@9T=ib~jV%UkDlc7k&MuaK~!YuHt;o&{nz{ z&mi7x97i+xMpED}zRCz_CR|Q5l_e>0Qe|E%cy3`KbX5?$O1saCIie2jdgk41!S*>u zJ^5>8to-&_L82r1_wFGj8|^ROcGJ=GVu5W4duODg;;g#eVG6i$?t4c4P~3_^VX`{1rq)4Q zOnQy)9Qe(R?A5+vr)?Ew7pa}ET`k4AAp!#)HGWLqDNpUE(BNC`{{}SJ+HZq_!e=du zPhJ}QB|okQoq*pzbLbV!61Qgn_OY-#@_j3f=QFu5N?DcrThFbkmMhr6CTci~@Q5sNh5?!Qz+bcY_Kl^KuH`an7RS7pil0l?O-}ONR5>ht za_4SC-W{nF^sW0e{{S2o{0iTuHlOde9EKulb|6}{dmiGAu0y6+ececB7Aw`QoxWd! zgiO;r?ZwXI%j;cJEXvKLl=iUY8#?GNn>uqexpLD(MM&LGsDJ_8y8%1Q!{M%|kNrkp zdVY52^+wwpt3im@SQ}YzS}jAK*NxFP<>ZRlcQ)(0EUMqFK{My9qxM5yAkD?m_G}Fu z+2~o8orQDgW;R$fi6>OWsM_rGFUUvcmB~aPe3x(l7fbMKnmx=Ng99P}Nb*>)w0?IJ6SNHS=dEoN*>lmp*E96;&dsCp!%x~&d z(PUjX%C5$R7Y4Oe6{R$HCi9LBiz#KM+J0-(SOxMYGHMQIkU#|UPn^3U(zsxD&0F|7 z`aGRbeRy>I5Pm-XIScq3vl%j4?%`GUngY3eHVMb>v?|rZu{B3<0^SiugMr;u2^(>A z()WWHH79G-qjD-S^C{X$E5&;D6o)|R&swDgMo|=|qe}bqO_ww>FywfzLoZ2e ze=nPk(S=`sQWjsWDr8iZCabY#&_*w`_PLXyP^oNMh$!~i7+fueHv64h?W*f8?fSyC zNAH!y?nG`Z%`3N$$C->U{(ZkP&4P(g+{mP4;<$7!I8Yw2jO6QI;8=eqwM^T@fY=J*Bb(ylHh)qSLvrHYV4PA1JJ z0E)KK$g)j%UjX)Y=~cpa?m+rg>uTx)s7%T{d(z#%<10HCwD9YkBQ;8A-eb$p*K?p<))sA0)8;N)|k5kkg4QoQCN5vsJ=9Rsrm-PHH?6u(D z*QzH8Hh@ zkO5Y&yw<`_?V*@IAC~SPfY^~h;%}{vkTV!vi$VEYaq5wo1El!?vfS@iB^tO-gDG;- zz4-?_PqeNvw^Y$5`&aP?=*WiU8b1k1R%fkhu&(Gahe+V-x`^#MTN_3lvq*Aa_kkg) zvHq=JL9hV@sG6nG4ycP|>je5_bX3_rP!gLmt5sXk&ge#vd)T3-mNUJP9h;j67oy#PSiY`A93=RUF+U@#thQ+NoPR`Y(zQeH%_GPw}W7BF{F&@qP-9vMH#Nz@tPpn?>{{Tw& z7oYaI6+UgB+Yes93ItsD%U~GeRK_aD^%@Fmhk~;$0;$i}j~eZ5`7fP(m zX|2l4RW@nL_-Om@#tL#7bV~Wy-H+ttoZ6}Te7i|i0nmJh9d3s}4V{2CF5Y8k?89jv zuP7iB!mCSV=cSrb>lO}6S?@e27ge5^e5QH;Ki4Ebd{yE3YB#-; zTF(P>bpC`*Yb@(b2JBa%Y^==umVq68AY~bUy{AK+5-igk7jH*{e?crWFV5w9-w}D6 z(|NkEX;jD(>oe$pa2{@A2PRb!BMZfnBo5PxS7u6fIJ9|IT^B`mc2tjMoTUk>SgVLB zryki2PDLQ%@!qI2u7f5deoNpKL=DNx%r@T<(*D765|sw@r$<>g-$HKd{am9TU@T z=0Gh-kfT(!@J!X#vx!fx*IMWOevSV5H++k(7zAXMwo@OmktBv#2b%{3h_8_}PGNUxWnJ3LRN zFNt_`cN;GP3vI`EK$T-vl?zJ;t2Cg{Dp^LYtKlHlrq}F-_S~b3;##j zO?suyQ=u!e*oy4RohK$T6RTGnhKshx3rC$= z1I76sUApyaG^2eRI`gc8CbcuEZHUA%l0AvyQE`kE&!0$dancuTWe4}jq9o#wjWnzW z!>_Ub*&BP~=h#x7?Zt6KUSw6KQ z1qv3wj#!PblCPLbMF%JS9PL+wF#?_zawcN2AUgZ)e6l3u} z55xx30q9F}ljsaMOZ}eW2q8H=zivd)UD@anD<^*d%of)1TGx?{D}(z=>Cz6;ncf|H&`9HGi*RpHl`0+e zeWPlV+dq%t-(2uZ1ge0>T?H0A_r*`USzZCm-nwG$0`#J%ltgiY{7AF1Y zDYLvY_8sC@FsxzjeE68*{Qx5jmG2rhq#}63Pq17=p$$UHlLhkXFpnigafDHly+*KF z-87~9%CM=f7xq802~aF?~5qlWMufljy1Q8VNXBROm@L zUg(kwQti;>JCBP!%QO#&RQYz;wX{6N} z56byB9$0>*qAp1?0?)K{6<5~-LWSo%Yr?S@f=xz{&|{>laU3xGi1Ct4fZqP#%AC># z3=jyC0ZBHydYH?NcpH9ul=p-%HxF`?{XZ7frru-3~QIjdOrlVD(ck3(& z3<_`?LWeKx(g}ak8%~tsQVCAeum~udbd?0s?yf8|62;$yg1b~J_|QBL8&$rW=f#5^ zzO_5`cl7xd)fk_Q@(3$8^r=N29W z*D@$EK@MlZ#<~xjJkW zIQb&)Fdl!Y?>Q&97F>%Ss$U9WlNcA7>Rx|?%p}!%l$U*>TUED687M_&y0|bmTQ?=v zfN=9FHiY7`R`l6>IckVLWSjovJJkosFe^xMLF_13CE}LjO|&#llh#;`oqRWmRV6Ja#1b3LbEO0MgfJN7Ii}^4Q{q2MVq)md*@ai68S>Q(O>q2OES5=|vRLSIOrt&1c}j~Tc&kZ-M$!-MJQdyX%ji0;+AM2N zEUk(|x4H`1g;u;Wm*e4u{yx;s()3EJ?SuGrwkOIBMCon}z6Tq_u53M_Ls@N_7F~o5(Ta6c6ao-0y;5ezU zkmXUQNx`0(nqz9y#NokG4=N@VtE2x$r>@zG`j{aPt*VInfEaJJE`h91ovoMPo}VrC zFGlEpO?2e{Q8>pl8|CgM->q1$y|HXm-*iFkZX0WZbtrrvFc4kg>;5!E{Dqi+ z23$=QdD;h?MY7)FJmHdTIIdVZo0#5QZ5pg;labE2q)6ucg0i-}%P|eUY)wsj{{t-T z)w?S@h0$umF1osRyUcO==H}zAwcHfFK3{dyS0t5#9AAt-0L+7)*Nz+YgWC$=I9rCt zqw=eICiKnk5&g``8V5%BzqZkxruIFIv4mjqNgWffU)k$Zr4sE})mdYQB&&E#*Epn} z>7x?O_SK|Y%+6cwDx~+AbjyP)mCA@KoBjYW{5ZBt4P6}{o3sa&l>2Era3JHRjEZDB z%gx649L0?#FhTbd+6{KwGk0S6CrKt)x(Y^#>gyJs_Q2QI3{A|CsP?@|f<{d)#B5H7?6SP=)?-~ynY>o@4_AxAXQU*1VbgR`+l5X) z_rowf+LKs~a4c1YWNxasB7s8WYC7zRPB@UN7&^sb+GQm>A$Vbn=ua-;bdk|3q+}WiEsn^jx5oQC_X!yyFa}Dzo3ob&a5e*jC(v9|K zJ{=P{aFplk+TYL49`;Mp8jH`)AKbLhN{<7pPWU{TiydaY71MzSE#j@_veuYIY<{0{ zQL7YVCmeZ=Zg?am(=~8mD`@fx6eGOLgbjc{QH$h(EQyj3(qbb?b;K8VxF|Ods|D|~ zOycj4OP42p`(E5{Q_sH03X9eA4<_w+J9Da^E0;|IL@f$aRM{KpL9`nwjD;$-9QI2C zXHqg1cdJHeq>W9wTDl-tVz(t7^BtE(0+`?4WS+WL4Xp^u|u)1%n=BC-v{P5%@L~!L{ctdz~)b;Q#*wKIG zVbq>tc@#HTc2&xjW?r}%zdB$PAobS5j-1$XeNwyw@SYy+3^%rXI}H?T!W%@4Mf;-LHoNzHT2YOzgP+H13cm6h&~US!#1(;|^3E7q5M|p37O+QWDf(%g z98zqbW*JuGYYdim0n7w7j#FY;y{joGNQ)8mQKfv0Xt!nwxudDWSA562a)=}N0)0>& z*0sD=*fpN8Z?v(14kd~?(o|K^UzvHZjukaj7P2S04oZQf2O}5=16!9DtwSAk1)@X@ z_}qZv6djwMJYeG1Gep1<5AMWpW@lkXyEZ(q*JIo|jKoCDL?m?rhl_(GE5kK5Bq1w6 zV4Yfk)Dd(G;>oUZ5`R<{|7r3)-)%)`g0IM&N zDDG+jZt&NcEuw{;@U`PK&?B}Wh?GUKp8~DUUW!*z5RB|wiIK43F;%0)LIjHQwP~92 zsh^H(hwl`Q`PFxrcfKtzF5q)sn6Q)>D=(0lRuCmoRmU+t7!MQU4Nu=wRWkt!o^g6c zMUex&ES$bDxpVBS(kG>{@XMnA0eh!N7Xg(4kuR8j(l?2<;=AX@XTD|Yl+Wi>19Z@? zG>tRs^+9K^&02-&{Pk2VMjXB@&8&eba?lMn6g2J}5qn2uOEFWzwS?n&820oe>au6P zEF^>*$93DO=3HgHehcPkK4%8{J&HPoW^`Afz1J0S@bUZE*BPdKHdw;kSyMpp#RrXF zS!?(lmAZ45xI3D4dI$6oR+`olg&IQ}iDHqlwm1dg*iGa!7uM||Mn_(>=!|ICu#t!; zs4cLl4N&1|JdKqQ&{jv&i>b@f>-x$Dfz>%1+p{5Pb^B_dM*eTUZL#iCr?HFG%ex5g z7$ihPS@+-jCB6}M*QA||!ck9@q6QCr7P`72+axhgRaoM;^{mWcb6r-+;&5J9MR5pB zrA_%=?9;3wP#^1l(uP`nM4JzPDEm2bxYSdIv`Q|7vP`xj$iRiF)~<;cA_^rB8Gz{SJ>P@YL(XN@*?f-QEr&xFPQmo zn?{Y&FDiwdgC}a=pF6pC12T~_!)8m1^(kikQX&*+WF`XJN{%Gp-Eei}n3&Rjth{Ah zleyI*~f-`{lt2Nx&EU*#M{g;wXKZ2g_>)T0Z#?~86l=81yOeGCK3z@6 z?SD$+Adkn3C!k|YA4BR;UgD~$=rr?wO~1NEgpv^bNt^d`$)Q^_7HhWL{Xg7}!)r8h zZsx=Nu`Uy8$s)V(Bx_vF5dg8Ue$ z5?_gH?t+MA<&DmtGK{@l^@L@kIRSopVSB>eSbPe4vY45 zUN=@J6PxcOq!7Fs8sh4S?xI0#jI)>yHHDkWZbQ}Wn`;wKIo6lN&>3~Ehdbv%Z9*&R zUQRemPiFNdPYuo}Hk0#wZn8so!vltnR}M>}H5xPNgfKfW7B)aYN&10NPJ9fJ3EXGZ z5)@V$ilPI@T7)b2fj)|B5Zh#5QQeXVEWj=CvBFJTq<2b+OfI)sV~KbcI=#tgf$?k2 zDX`X<*bzN3pJep7>npCz*K&%W?AP3Ps?ON=-zWkTUN|=qI?HU07g;XKHT=|r&+9DN z?F%(p;~UI55TnP5PQ>UD{XqzLh!&dM5y1^IQr+OSEW6#*6|pL#lIff~qm9X02ZeZ< z>1Cv9`!p6p zEV>`=?r0)`3CP{lNR5B7iXU4ol9EjRMxVfP^0`qvW9j*7N^fQ6H)meQ#JQu&h1Q&E znR9Ez=9!=2-}9M4xq8+_my2shF)${cg zpfbJ7?hkL;P!4IG%I;*2w`T2A^xEImjev-=?z^!Nv1&;z0u=P9^OVZC)u*4z z9^PHA+)-~+AD69iRj3z7-me`W@9)fCiu~9n0`~wne(gHG{64tv$GPLT20!uM{yF*F ziCCZdGEU#4t0nW6Z*jx={=X$6jE%Yrys_XP{!Z&B>Wh$|Ui>c%2yuA{eS1GlE|X0X z-SXFPxfar_*v^f2TDy@>;WqUrluQQ}{C@y3^vYeK>un3S2gPq)TnIxG;VrS3G0;OH z-#3@Vjb7K-h&V~NctTPi-uUp7ZpxSJN6)%4i*Df=&Y#`6sP@%sHdB`UAX0e#%N{u! z!heGq@PQxc83;OU%Of5Tln?t|j5i3~uRZ--rmRIA5qj=XVRtdz#7Pi<9aKO>rHc|C z0MI~h$LA!2COlT{aPVU9d^~z8UPLp>gAKbEuBO?a#Dl~)qVX>JSq>p%0G1n_I5rE| zf$_01b+D-IlcxOqLtXq2z{ch}zr4;pDl`^bNJiU@VYh2VWw(yMXx#G;pt`yVAHxPp zG0j}EeG--;I9!mGhg8yF>>FRS$&4x?`GO*ri>emCVg|ApszFI_#m1`BeOS%5H|8$l z+85=q4>y=-(#5+8E_7s%L1UElK`>>v7nK?Y<6LoBWN%Qk$ZSt6#crZ?BnpMzXAFz6 z9&Ge8FsM+n0e0@zb0i($T`X2h2OAxb!wPI&z(qai-k~gT!QtQr(ulrqL z-oHDbH5$a1kc0#_1~?h2vQmxQ)8;FTQ+8fmeJk5zD)Q?Xyy!cF8u@^MzZVhxlE|;v zNv|zN(xGXcm;Gb*Jc0vl3^2x+ghi7RQFTh86uG0%;=_kAvRhm4F9=bzsA~-p(>7JK zsf?Uk8=A6ep-DEszu}GXySo&JxU_M%!W4ZqlDFkgK?F2jEQ{;yY2ufP)r#dVzKI6l zJlop>rA`Iz6ED#|JPl9wrk5fz{{vi~z`gN+G%h!*gBDIh;;hlN!|@6V@R3%bF0gvP^gquj@b8xT zDyA#!c(8t=@sEKt5WNr-S6a^qGQGK?T*5>ZPQg>H8?P2#w5@KbFNW2(or8%uX{Izm zP+Yo57-4A`6oUOMVc}rH9l(8z4WTO#IRUcKW22Az(gI)ExD>P{#g8Wy6{RBNj9qDM ziYZ4yzj~Uk;*z}^unKQ`wg4MzzKF&RfoqgC7JG})Mlxk`B^)@58)YU~XGgZ9&679nrFQOxROG!O+lg2V5 z|EYxsE`f%m{m%3yv{|DrA3-%p2Bx9#q3E34B5-2@|5E;i0fg2bct^+P%Cs;j+PwsI zH3|AAmBq^-z8k8en380@h4oE$oZS$UN$%Ti=r0PNfHhP_lEn@&%GgDl%v?=TmiB`n zJ>+FB;*3vASkz1`@#urUI9z7b;X}TVw|#rn65-S5YD~v~5xC~(z#+NAIY>wH;?-9f+2z#5M$&< z9*Ubf6m0k#mv(;-ByoL1VSH3f?G(=+?iek}2MR*P@V+;xCNt*zZfEyz(ISkOTmF%B zDELKjr06~~G?-z8!ukRu9uy5FM(Dwb7R;qk8i{_j8*RQyKyX$Cyas2-@wKi-fRkgcT9Jq1a`ELIYN^ z?M-rKCX^Y^3%HtCRI_!P6s<9v1^m2X*29(%69rtCtnZS#fBJ=l$_}G5;4izIR2dQ{ zh}DMrArKLQGm7`UYDWo`r8^$9dHPnTu1LkaDNyfB)dn-QY9-;f8!kj2bAg)L}&XsLEs_|{w_X% zeiPP$6=8{3YYTBv}yK7&Lui|t5%KI=2W$7M+R7~xNGJ&x`+R(5g zoG3JO>R9(7?=^%rIV!8UespZf4~7F~IT0h+c%%AhDm8Nc`O{DBjwoX?87M9RwNHEY z5qdY18vCWQ9;xUIS&k1B$r>K}^1Dki#2Oy1sNuD|mn}=R5Y=f?)Es4SGtCEsHb*1P&ze86O}+#?!Na=m4mv)y}tP}xk<-D~rskjspv(5V$yu%Yphr zf}0uJpQA6j&D?`)7+{@P3ELL!c%o)}Abn;Ew#4pzd6SBARCagCrAKm-WWvI!O|93U z8+DB+Q50j)WZ>_A&qf3O``0fY)o7_l_}*DatmiiXCp2q(Kdvc%^&s~UDH##@mia^| zKS}hhNkugKPK>>ry5o+-4r=pW=(-Pne8~^I_F7jfZo9^E4=_E+4tI0kk=9+u2#yxw z{hLm@@mdCdSQz??o+?%uNfRpa{)nD}LYV{&_M|{Hjg!d}|TB?QR!r`j1_}@M(fm3^`G8qp*n?08c z7^Fl4FrQ^-98A<C@ zokp2W*xVg$ffvXeHkWdv*CD_+R_~QP7ppR!e`MYpHesK#BS2M!E2{!5R``N zP)n~e%3p?3;r}U_XXaj2NDe|a7V8T`ol6T8lU}*_NKpv7`1lh{0(vw$hvV504Bg%ny|U(w!r!{O(w-j2<>nkO1=*uB z6&DX2F}I;NFjH~MU=G!75!?5y2XJwQKObA_YbJGAs8u|t8}=v@IG0=uT`25d!v-wWz#z)9gzu@bh>E2MvtSu(rVYr}`=1m9WT2PZVr@&U`I#|cWN zP3#>e%H})pAsYFFSj_}u9%i8N1HVar5(J6kjzvyxpN()!^V8{3;i&Xm_9QsBR_eAw zNvF7SOdL=5Szjv6&F5Q{nm$)J2_WO<#CXBXb^O-1aqUCX^(Wz;0?O_L3QEU{BT0e4 zDxz_RMc>V*!wBQqOuq0Xs&_En)fY9dgDRHmZ8|0~G!Y>l{~i}mKAkY$X*RRd zaP>?3wEv}j;S1e__mkJI{=}jABW~z_fkylCE_x9@e0a7u<%_waXCeWu_8*ot!3 z2t)O2dDlNzJ_*5l64+oAAnc6(18|-?@Tk%8Csg0 zfC>CQ!S(4S*Kamo^Ng1X-3$WZv&Lk3o-))cbf(|An((_aYR}H)`)Z_Y@p~eUOI?&8@8=J*JAYWB4#3$XfTjvc!(~d3$sJ+*+2mG5h zrY%r0`|VZnV>Ku8{$*kpSGK`{Vb<(aZ|+X}l0aIg^y`L8P=JHpN3<&a(os+5t;A?! zpvn5v$$Zs z+mHs$rXe^vojhRX!Zo2|AcvRX+EW$JcQ7N( zhywl9DU6(8Eom_1WqHsTR!i<6zkxwPb>rAt1&u{|&ILgu;r*1-E|5or`)1}kx4MZOx4uG=0EJ7*W2V=M+c}7tdGj%L_2R5fpnJme^hfXQK*U)kIH+vy&#G zr!F5HUs4TEjVAb9Dr&iWS&BV$9Hi}Tgua1S?1b3RU|-sE_cdLzokzA01(G2fGKQzvDaO6Qxg z1sj5=0MSU{rYv#2%+9?caiYk#qnB^>Em1;lc`5#k$!lZlZp8T1=i$ZjST;Qqn$wVq zsqgA^1HuFhA$NTNc(P<@Fy-}5Q6-&XQUR*lAiO#lD=*e2%V_qed-G)d94! zoWVZ;Loxp9B`*O(zuLPOCS$jPY5khR>5*yGemmjyR%yo8tufL_8ck4SMOg*#1DzF{ z(*5OQEi9X+&NSb5|2$CjDE)}g3t#LkEH2(q6k2Ah4Z5u$A^}{UV9#f65HPCM^4R%@QEbNEB z4;fX4CN~eri>3@bI$>i5q_N4Y$K(u78X;IBx{EyyXv`P947UzK#r%quHldY~X2={s!=Xzi$l$)RJz^#ZQ%RIbIl$P^(3u z(ci9U)Jiufjvg=TD?pfiAesO@ee%hv`rqoU{OCEStN^I}wdK@w!=rc6FMKwkYdVn$ zQuO`7KSf2a@~2yPPBX+YP-uRnm*>&zpnewykD;6lT>Is>DdATA;N%y?nd8T`p&d5y zg@7U9ezRI)_c{Ws+rhR1%gVw)5(FvMACxfgg!HAmBH=~+lmmw6cqT}M!&6FaVhzK0 zB2rpsf0RZ)9U4Nr3y*Ntm8yY&fA|J5KM$hC8p32@X|G5Dl3vtQC#(AxhN(ylL6~@7 zXyi~Vg3#vAi&JcqZXVIhX7_lfnX=(FS0yP zV1;tM#@l@eOazq2t+ErO)N#+35)pv765c?8xEteJk6Wyd zAtL@$tU|)sXKnNgC%0_rk2zsi(^m+BHj;-M=>LPfHY}bxlQR*8w$m=qwD+@X&#l@% zj?Q-CBp#0#$A#i28Vj!r>YJYzImn5IN@pQ;-E1m2kO2X>X4FYrK91qRIsQ^wU$b2H ziU|yt8W#Ay7Mf|q%>?nSj+{=lKdXW>pgXYaw|S?$d|&Zxb7_`cwc@Bfb4+%lXkXC! z&4`6fkc&2ukoA=dDTfL-Qluy$>1&c1_LDbVPK5Uzeks2+AjoYNdp%8t+k9X~^6$)(wOr z8Rr62^3kl9n)QB}lr0f^{gY|wCJDcEE+I7kqI5Y+qU$zPCJ35$<$;3KU%^)5z#0c- zdE-HqWJM6ez*w72Q<;~3Ho7pxjaZsP@7VcNr__f?lq3q-T* z)Et{u7PV<+j}!ZwtIw|~m-fW$P?qeeg=xQu~Qkh2QL|s@*j}p83Kfbd;jMkeV^YB)-j#bsY*wwu-RyHrY(Ah4m+gg9@sjbYU zG<~E1%rsQeXsM+XS|0x&>fR|x(qP{gZDZP)wr!i!cF(kJ+qP|crfpQ)Mzw8o+S7K= zt#$T2d+)W@zArcKdAKiC5fvF(UqpVH`Nt1OUA`>804E>$W?l<8LqUgB+~w`af}|N- z$ReI&?DM{Hky|4J89XUIBghPX)vj-egInzqYgBu(l~a@{hG&N%m;7qCZkOKJt5^&B zf>x7r9}a`hDC~#1ka!^>L#txce0-&ZxnsLSx0uNq8LYN_w-LufkZp{PP6%-EQsTNK z7)moh6h_*lXZM^JWe*5)i~2m5a1ZbX&Ela?VeA}Z86x~Xp3~e^hKg6#qa_Q@Vb5rK z)&zVcW%VqmmkL-DMuz^92=IIqpE;(zx7JA84y1JhV+g7wFDU5>q`5UK8*spnLkJ-s ztQ>RN&=x`p%f%G;iFRnZ7+f^+_;ksM)hrmLAf<^m(@W^Z<2>bv35z%Gx;Z!oV-dD= zg2~LSpPG}JypSgoe8p{|&GsDMO0Nof^rNT>(_|Xmm!bW)}mcrkz4vz*#>DwL+Y6`>;-R-NA`lZB4|8dGL! zsz{gQF*>v&Uvjw8JIRKp5Xed5>Zm%@6f(aQXA~KyGmvI3y63?+xsBKjd=*z_3n$CR1PJ|ZuVDhAJt+}AJ;Jz{@iB%>YLZR zp;xYACy_&95VSxfYSEK91u=XmvXx5yYWLAju_?x+9a-$Lnie~yFMv<6e1Y3#8RS4)p3d9();RTX2Dyr zqx$Wg){m|ZR+5<7022$quJo8#B63A$acKa-dTa+cmKo%9FP8o~imkA8~1`cn3k>z|u%oM|VkjdqExsj$K0;Q__W>@n^U>CHR zC10QNxnjl$L?MD^=1ef4AQRPD=ww~P=_t^J#4qEna3<%RcH@NbSO<%@dn_qp!pF)} zxUox7+5ahE=J5Dk72q6UM(DgBmp%n%vImHcpj_Ryyk(VbMPXu3fTQ^8)j|4u3+84N zJz9O90*s}-ui^UIMZ}|(sK^}gCi=Kpi2BE=Eq7^`bnHLYql$FAJzW!=7S0(wl^dIz zrs3N>E7$*8kMY&$m3X@O=msBaS=}u+TC?pyK)}DW|0aTO|D{8SILN!?O?m&vs&%|h zKY+dg*}06bPP``{iR(q}eCEiVv~P#`Okj49dMC(`1|tAn1(7S6-3Q9=ALTj3wy43U3>X*R&Xea(U+uX!F>XL zKls~X+kp8m7%~}XtG@THmo3euWKF&F`LVp>K4nb&1sja)1<}ZuA1fY2<9rLQv56B5 zBm+7u?gUnuzM)XR4UZnI%oSW^z0YtV1l6tlZ}pJ>5O-pOzKwr=c*=Rp>SdquJMPsT z!ye~9SlM>CGJl`pdv%QOt?*@d*?1o%=<&I&iTC`Hgf4XBs$gObXE(mblcS5&?TT$ zy46%WHW*sYYfI>GBLA!H2fzHU`R{1Kh*8W*-j3yZ^;(B(;=x+f4YUromg_3(hXb!w zMy+5sJyUa)fv)#dSv9lj+(U+4;-mk1hq^)iDEPnN{kFI>wHzE4`rC!+y2IaVC!Pr^ zrUc$;Kk27JUVIMgrcN|@GcEolncmgK1}(CRO|t{(XGJ~J{paqxpM^@8 zOE|arA2aWEf(gEWN66b)pQN7&pI@0jC=1>ecWouMQEWP-R*)G+o#Fuf7cGfBMjRD5 z?!8Gd5)@*x2vH1SB_~je1+dI$rmhc^u9+R!+7X`Tv3;IIYms^sQInKh6}FyaHd2Lr{bx&x)D8q!dBwNa&Wj zlbDJ))nl^q{2n_VYu5gjyZ0Okh6$@1>ZFVkzN0J_>f_Vm27J2q9)3hTNf$Le|Gn(L13cmp5(rWFGh#FxtCZF~;*uec!b~z!)rLrR zIx9ni^$6EaIv;}6hYRx+yDGU0S!}xgjT25oxCdZgBw4eI zze|lOD2yev{7_gXNFAAiK>V6ejjF#pQXudEGllC@CZIcsGNo~#*jGS8p!G6z`5nRl z{myIYZAo#-)kA3xi`=9qdUW! z^{~o%Gv^-mo~@KqH$ULjS+nqBo4q2xw7?%sqH<*Y{YO;sB8%N#Vw~> zdJW%MslMas7~2sFjxcBR{Q;Hm`W`Djg3e(-ucjOY3(R~AI;_0%ZoN~_<|&)hi>Mt3 zwt|G?SP0?Gbf8=9Ie8^)_(GC0wugQ2v{yyTwp4=$v>QNB`8B`T*^BLB-wjchCu>W zzNH`>7dA@M8q9BVS~wZ9neF8@ogacAo#&F#0N&le!jm0g`d zQd?znJLg<93%t2dN5}PIX`z8)bHV*D- zQc6#Sqo@+}ceaa4Y*8d_J-H|CrlwZ=7vz96^LowK`^cp6>Y4ZliLMZ23xW4HrM>S|~?qV~4ef!OT)v z=h+Z-`5fittJM`c*ciHkt&xH>C*;r}oCR?(o63<6`om;GMM?b9890xiJj@Kpm~7@rVte_< zGLRA;`VY+oDa?|R1CwL$Go&YVI5Cy6`k44s7k@PC;@BVE0?3?-AsUoyHj^>f-+vP4 zp8N%KK7VToct?9Ca=FgqvwoA}Ir(E#`4_AxU&lw;M}J;^V%d!!ESRR0dO4R9#+{u+ zWFf{orJ6*F0P4$TI*k!=0_!)irh`lA(UD<#YzdyXzQZ7z?O^ow1+>iO+sAH;fM&3Z zjq8N`-jI%mzBh8B?}D66nuQOiY!KE}N#4!!0wdYoHh?9y@r=ckDm{~>-1Vae`-hTC zM?jU5nnPh_MF61Z^!)cs2XYTTSn82HY}I-Xv%K>+wBAaaTunSeNHAtHiOfwRA$`Y? zcXKcj$|$Lw)GwGJD$?tU%L(B$QyvPHl?rSMM~(07{#;hqW1TGiD)j~sUOrs6Y-bk8 zS6B)IUa)nR%NO4rtCntbcUZ<)uL!)98sWS`5GiqX<=zWWm?`}E5WR~f+gy{!V7ku2 zt1+XeQhrf?Y8B_p3kv41Bg)~TET?LGpO7oY78|+148Y&L-|jaU*J_8Fuf-Z=`IQ2* zugK${3CRS-+j9?{;#w7+kfYrwQ9-=!G2*d$GBN59z#|YbZN2gr?EEJ5^DT4d*%Ujd zV_JSlvw%}7De_p}D1mKJp z+xrqTt$~|fCm`7O+7I>(juZCAdXRXm$L)y#S#7hXZ6F)ET8rA^F64xy!JHCqSVKVs zQE3l?0$$jRFiN~H-m;)sTU69Hu>84$QGGCn=>dPM&3caWE1`5vStGq1lgFLrpb41zaKu_Yi35;6`R5 zCPCX~TG(!;76rwRZmAjo9J9IKbV3Q##LXMC`kux7IQe^T{YH0E&4kNl|AM8Q{r>24 z?)B>cB`Z(LS#(HYR6NH%xoq#Xe7UoWY{>7nPCUAtuhVNm9GVwll_A@;e$TfwtT
ybEQk^6#DzoG13Yu0+G7RJ@jPvR)|APH^kgK!2HW@|1&)$p8 z#A2yMMk7FfXv$%fJ0X?Li4YAhlmJ*t<#n6F9o$jIFM3{%u3w60>P_om>yxhApzo+>!V`ZGfqry^HH z*LG38RQNJ|*Rg5G&1Gn`?*pu`O9Xo5tc%`9Ib()~F_7mR$c_$@k%++jG9I_7BbWX&HL0%^~V% z(qqd^niYD4QbnR&$24iCap9bN;wZ+f0DIf$=Agx@0%KcQbcRd1~p zKkFVvKQzdrZiNEIZ`z(iyp&Q2%00`g&5u3Sy`spsq z@L9OdBQkhKdNx*_9j)U>2mJ4iyD{)xha&i_5jFuf^29yLL}A2yNig|+Y8{Frn2{p% zAO&PFhAr%JF3)JFFd@quio6#4BaY>k;aM4N{k-0?6Y}4%^Tfq}!6^J3-g-aENDDOb zociuA8dHC^31c71Po+$k;EdvwVo;Bz0P!xjI>j+X32^sb;LWt_MJ86TP%!%!suC@? zJig4)tww9?=-Ab8BzU!1+dzhE2G74g`jWF1Ylrg{U%?TIiRN@_p&g~7?-hj&Z%dR! zc6B*V6VQzxRjx@ab<YGhU^WdF6FduqV3D%fr5p|6^g^TvR z*yTD+)_x_*MgK}|kthZu;lbAHf-s6#x+nv8E~g{Zou#8vA3{m6y*5R_+0dAeav3wG zQ#s|m@#5ZlKS`1q2HiSj?@+tV_$8&dTCcIiKDir~F_)}_sX-N6LT+A-|5V2Ta1LUE zl-SIi9|Wn$@9j-2eQE=pwcaIN0GT-ANq4~p6{YlP2M>mU+XBve8y!+PGps|CtG}Kq z_ggp+g9@C6Ib~o(GXd6V6)r%(!zbMx)8q5cQx=2Y+!>SyK?-$Vf-Br>yr2lZLmvAT z>0tZ2+tj0?eeYEY^k;~dvpx^z`cul5i(g7k-CJ>gz|U>Lf13mTL+_rtNKk3)&57h_O2 z$1}$cQk|x{krJAth?!vTCDk@0ZX~{M*&KI2>+nxHStIQNAHNf9W~J~7j*zcUYgj#) zJ5w{DLNG)%4P=Cyz>Ucim-B4E?vT+F<4KNCMua<$h^~g(IS+2B%yfpFnEJK5(>;E( zY*>kXTww<@zW`!~ZZ<*VXQDw8ep&OVVycAr20m0Ub)^*7= zLxz(PDdMf|ysRc~Z}5i|wGnTqPy^ENJU^}|7wd@Z==0BFZ-5)dFE~bkZtiR<=Wn3( zWOn&aV=+D6?%l_$5#6nIOzpJLmw`{#zhDkUvOxzj{jlU_Vc_f&ew)Fsi9yInNpx5i}~RlmV@Gb6XUGW+P=@Q)oz`={IjFQ zKpYun?RlqW3m$g`!4EoK<-4RC&I789bR#^EdfWU33xLuS{Ce*#tREluPVBN5?CpAYS@BLNk5H7HWvR zn&w6XL;2tu*EDf37;H1k|K6~C!s!wG*fJjjcKTN@zZUklIa3o#PVlu`r^U!Fd0dR2 zE<$t!hzomIj5e@l!3oLIn$cu?9SN}P0%nbzN#aF^n^qou&R9O$zcb|DTsY&**S4;t zM{&R7#Hcec(;3YtHL%_NK-fxWh!;-QVT_dvN{dHI!lNWZaNE0}L(PK3#FD8J8`%)f zw&%gzmK%tuEaGIQ&*!wP8K{Ct#`K(lS zU2ZGc0l4V~p)igrG8E<1SGv-bf1Y!ZyA`lSh42A$>-ySp`liB{Gu+taQm0S?D8l{G z=&Z$0bNJLhaR@)Cq)HWQETlH+OW}y1d7+e*ToVkBj_BjMM5CJ0!AGYWvVvYRQUkr6 z)fW)D(O8d{N3X1TdUNS!)cjh3N*oP`X}_7>(9ocpg2TxXANLnbZ#MEzDO{c9h(n?| z*T{gf;S$m08FndA^q=%Kq^azv&fV4`9^*db!v-Fgn$VT^I(rT`4LjHcs^*xzpN2Bp z;&%E!R3OSKpMR`CGw5}?M!n{wHo|b(E@sYoQN*_MLWO@*dTAVu>vdHHHcIGKzW)Ms zF6cVv#eqVOouk*=OpRsn>1u;vs#8Ecj>3t_zPAuhU}z<+b!BGF-}*ez0o(1(Zo#Xhj%H(Ds^|I!JEMfqfJb# z$V!)C#C8L2?<}2E4zR&4p37av=OEg9SBNnSho_+}2OuCgXYR+tabOWrp$!xKsXtfc z2!(gSy>==ol)%LiFIMg9MQ6dFA{ahH)*l@giZ=!WV2Mq($SBR|1+!61xKFEU8c{LZ zip*;u9t7ccwl7KkbZ{>ixi6v-{TgK~@5n^O{iQDM${6eutpg66Cs$;eTkQZMID8$Q z-fXT|3KdgYEK#p&f*URV@bf0IRVoA(v&?Z|x&=vtnU{b3 ziI}LonDJE5>D#r#*L1ZdVgo}o+D;`GBD1UtzliDd&9$jgwZwRJJa9DAR?|DT}pL_9q zcwE1IL~lc7w1u;9Jj;psh`W%~2uS9P#EzU4;=yT+Ibuc_4g+&+N`ItsfAMQnOa9iE zN-@`V5rsBn>o`UJ6C$GlU;2yj9+Jy5vLv?Uhn#htoo6y-th+>UF>bCCi5uw%t_494 z7CptmQJg*{Z)BfCJX|8AD~ux#Nh!Kvcz-I<)r|qKU5>QP*VG-yZ`9ld zPcp%O!9=?1)w>@|O0^e$)pgv)nKUKXq=qwDDk|Hsc}ZCSetF?zNz0uyHZm;W<=qxp zBN^YG6y`(`!Sn}fN@+%-KuMR7rmv(p{4rV3VOxIPU)UsOX36YR<=;2U*ny=;N0%O;r0R`^vtL$_yQsO*|-s z8Kb<;Q%z0o57qucVq6>NlbfX+rq}(B)$|-4uo!G?x0B?g-_$o7KKN#$g87`E$z@LgY18l%oxMz&nv~S+?+u052)FRUj33UW5@bO zGG~rqVZ|h?`VSPt3b0-WwVdTztxo$Nq`O0?HC3^~_*}rCKJlcwmU_DWqAFVLlHWC2 z{}C{5pD%#a^5wBVb@3;XZ~7u@D_^x26^p)^Qgyr*U)k{huX1#l$6s_4$r@W5=2occ zBW=vc(VpwUZyM^#+wt)Ma4}^GxoZ8`_E#cxO>R`G7Xfv-xB`MP?23P^^(lc)HmhY+ zqc6lptny42#6NzHn1`2Ivy-Kvu(3Vu!_`yGuf)!^55m^ymB=MJraieQuxL!3OpiBm zX(t1xu#(2653|};B4*G$b?1^3ii>jvDO7(!=L(UD>_2UHmc_4VO@!5TtEG0!)pq=9 ztwrClBU;mne!kOtg4J>I>GXdjmFT$oQ z=Q^&lDJom^_vj3e^$s5N7VUR|4T^R?q^z;5)1 zA%BDHUrD@%oLek<9cK|K!*N-0(vmbKlW|}S^1KuT@&2cPMx$eC-}9@L!nu>u7y&R? zHCCPaZ(C@dyWcb{C~?tCbEEJoR~iX+yCvf*B(e&l6VCec^mJ=<$S)uy$n{`7Cj^rM z%@;aY0i%j>(YEW1<|kIx89-g852z$?vmhfoc>||3rY5(RDA(1rB^Oy1$C%3El9shC z*T1E#-Tm-w?sfH%sv_Qa+yRDyum{k5CWEa!kjTdnp z7!79BTyl%qH(aLSZCc-+Cx^z)qXQsX1XHQe+TQyOnI)b2 zo<~Lv8Q!u8RxE-2Q1>-np)n6!wv6`uw}*|0+4sEwoa@$5=9{^>xgvwb+BpxSO^b);t#Ir88KX`A~DH$g5!fL z=@xBS<8Km=Y@0m=5r-|R3-WK#Sj!3|wrup6Be&&!$H&rzBfQkiF_Bfqq}QjaRlOCKv!5ahj&sN?7Q3npmgejL2* zyA@Yt%>Z2#`xeZ$pXtQ@*!AVsk_QI&3%t@1B8%grP*erFL$d9h$5hyBWgU@ji*~4V zB9q9C@k~A@ozg)e1u)6Ea~?`sQ2&XUSjhAELATTM5=72%JsOB+>iFM`1g+7VN+DNb zxAkwEw1oqxNwT*%@DLDVU4}`0vU?YS3O`CQECK2m=oO?Ua5TJ^SQb8WlX|*w8kWIP zVf#LHXBZ`MIjfra1J_jBA#=GK!ZF)4QLZkvsDoE7ZO>vXXgAbyk(2w#KV6#;34`_* zh4E~Mmqh6hMILszF^Cjm)r_u!x~ve6Vl8Q~?TzoO0PJH$q&&_!G3B8)Y4E%}Kq}40 zSdtr7cJ5xzD<*lA%Hc04i98P=vM=gG7wY#wM?dS$tgoL&_DuRi&CYgwY1V>!|t@g3?x|dIR6jKS=z1xJ$1(JmgeBp;4G>!m`W@we)huC%*t@3J@3 zb=v3U-M?T*y|mXyxIEbyr+1~^$r&ORP$qbHoczEvHQ(?vCz&)HNZYL1t(V{#PI zN$BwC%o+&{-XG^x>V;P;O0sB<+^PDR+;)yel8nG_jkP@|p74&$MBpq`771*M{MqoX zBvtO=ko?{xzcC7lI&8cz@~XE6-RerB)zj{8Dh($!2hi_-)be5-xCb1^9JpCJZ`zlHoE@u(^w z(jz)6q66N=le4F@Eo*bZJomx3*f68hh!F?-83V_(2DQaV|MbUUr4;$N50539if2@C z7GWu7bGUzC>y!5OFmCx=KLX`ig5{W}SA~3%7$hFWvBQwRWl9{Ypx8S(!S9ha`UY_7 z*6mM zB+K+aGih=wrmNkw$XL1qFZ%Z1zdQ~mpkp7*UZ<(_lKj{Dk`i!wLhmRnN|=p)b`aJHtsyuJm} z^U_{euv;oC>;joTy@{0=tc}w7Gq>wJhh#C_^eyM?Sh`ld@f>|0exeURb-HfiQ~Stu zF57=g>y6}1o*_W8>VAxn?QM)#eK&A!y@4hAKQYB3f>T$&_icz_mJ*VXQ<6)(FI7xn zV&vYW9p&g;|ZDZ-`7D2rG`KorB_%Lh+Dpde`1mKsj}+ zMP#d3dh7+EMw%lRyOepe#7v!*;pzIqmw>($*~00s_SWiV&Kbjc>DRiB+ibr*G}n0B z7o{0QOL+kO5|gYj8JIMWZCVGfwgn``145DwT@3PYCO2UZAEE3!azt}Z z-!WQRI%s|6q@r4C1XH}1f7wlgm<_oO_WF$7u@*%A?!t!6P7D_5H|hh2xKwubk|F}} z2PO^h$f81;LlMNO@T7j;p-gc-%FV=NcpZaN(B7QmQSpqTL8DIbh@iUQrI1mEAzIhw zD27q4w6f@uKv+tq=~$^4Em|z8aV<~?2_8xU9bF|JU2JpLM9|jfYm9(}lXIc?>V1kq z+m%W|=qob;nDWalBe%3Kg>5mOD5n z0wZR+af)>@{QO=xrO8CwzMLw_R$u5%zYcmB?_i;hxwDvNFPBi7CE575dBxwo#tgjA z{-Xt_J0Km`_S0pPcxX%33<)n(?u6XY@>68Lwse65j&NP0N2Q=VS;At?G}z_#!(Hqe^vVb)qYS1)#NGZKLfxe9<+3Z3d_`c#Ov{k}k8jeKy^oUYzUyn!Cf+nz57$edfI5$O zm{`(gYLy&h{Ren8a)N*gr42XyC2cL8>XHY=XdgN)otj7e1Svr4!fDyl-8oW^s-f{= zQZxTD5#!rowR(mc{gUOJ+CsCemRrUWf1P4#Px_ACQcLqAw~m2?C9csiF)GsvnU;_P z!H~!emi7qQ_bxl=u`58XBI2VCUX7iqkti!jHmKT`EoAg`X9g3n(ts{KN!3LEe*a!1+p0!||BWAs4Ye`!blTmt~diQ8-&XkqeDI z-@y6EEHKffz9fN!QYvJDrrvdGQ-pOK)x>0DV}1Q3u138|E5o*GNUkk$K8nQi0{e@+ z$v{FT-WS~&t|7>#)No^oL8WA_LZr&c*_M$tiXaYHZ1dS7%-r?0H&epmO7)GwVU7CB z{GA#WkjVkV`b8dNpO;-MgkaL7CZI7(!1LX#t9HZus8;__F$wc>eKHfXO7hx)P(0DM zaUxMdjuE5pmq$Wx-0|HhwIM6x=te8f8{Vj+sj+=76)9$V_}DSPHi!D)C7;!E$-|IH zDOQ?5nj!1-WXPcW=!Go}ibS({otHzE^4{~(aIPBdd?1S9;(f<2!|Gou>k#oTT1cd6>S-A>O@Z8n1z6;F*={_g>~3k2*sXy-DromAU0BlR@}Ab z&$Xj5TIb94+NxAk*Geus62GvY)#n%b6 z*li(#l=PlXktfuuLwts2?dsDGEJ2Y4+1fS3sfz2!Cl>{fBESmPUfJaj=MKgK>U=}H z1@A3R`vz=xlB0cl)P`}_W*a3vek{d}kKh&^KY9m-fCHmh>Z8L>(Nd>iCi;k@GLJv)M zyF>hJUoYT;9#Xm(6G=)D%%Oo>xZ0)y*g=c$!$p-!Iwc8OSf# zW18md_ado9}V{a~#&E=@UZBvGrx1!PQuR!Z|%X=1UE@Ye>Jia^*bCv!1Y4LvjA2wP@ z<Fe-NPwDjO3>a zr2X)se+@v%1<#!(}D88Q;zh$Jz>5);!82m0tFH*Ry!;DblHZTlp+ zDBZHt3^eGjBo{i8DU(bsIxo#?RV#-r2!G}4UssckL2;2IQRn)xtE}Da$mWW_&$~GY zV8H7c;W3oMX4Y=?RIc8^@HV#>BBu3Snf$>&wb=HA|ayi`^jUoLUBr1oyrvRfg( z5H6DcRquy?eyx+4LN674EGQv{qhKWYJ*ErMfO-5T6weFQ57F^P>U`^qr1T6)hALFh zW!*ZNuBQ2@5*9-iS%3A~_?;Kmb^f5+({kPmh8Be#DK0&;1&CKbn@BVN#HLALYyvfo zegNP>U(z25HgDAevq#i=vXj<_{*_EAMm8=xOZrPh8s)#@tGj>1SM*PU+z>bCr{sce zGZp+^XlpO7rb6nQLpYLAqDGa?87#v}nVJk4s8Cv)B}MlV<0vwAW2WD$3G~p3NOVZz z)H&LXp)*5fCjN)_xCb`MzzpA_hX>!P;d9wkLn}1DRo*E$(Km3Bj z?WpDeMj$OoDi|70HU#Kqz)TmzOdLo~SkHuQDIz55PmP+3C?zg0St(yCqO#3h8aQj7 z6^n<4efzv=-mY1?U7oIeDdf;W98@WFSenG&#ogDzgPuyJXbjNnWQzwSOU z$~iA@{-^qRPUDjyX^ezPsyd1WF^@?9t&c*}hJE*Ho?Wu94w5CIJu;G^IF+3mSh~q8 z^S=V}Z@tB*@Ml)k3(tF6rUMDay`{;BU^s}3Q!FFj`xw*(V@)9JYq@rF8fNE#>=3C* zo!H}vhTYKugjdb+s8%ko?mqvXF)Ws8u{l+*)pWdM728V0CLVQ>M%_B?{laifR5~uJ zXF)HF6{mRFcml>)J_CNUC>)&BR$FGPx49o#>os8}p(?PYMWr5VP zU^r^j>aih?)ic@kybp>DWx`y>S{6?ULrtxnG$l+zm&D_?90>(>8V;^#0luXGcTZ>> z!d=pAkW=h>MnyECX*v2{ce2?oitp(+5tmy)jWs~zrWdjrMR((W<$hui$p=CtT|T$ zA6O-c;X=HSC{j%kp=bDMhd;_*edBROU9S*&!FH*+`Vo|AOT<2&IqpKGCf}lPSSp3U zAJ^Z`!+1e5UyG(IYv$K%bTV3J(_gH;8U4grA4A2A|GNyNwndwF)VX%NZ@DWdH|SlK zDzm40mzTw7#y#d(LVyS)8pP=h1|Z=fkTrO(lfYA&BkL136!F8McKKkX`DeauH@RYi zn1!|fLtL2c>Unq7R3FwXi+6Z_mkqjIrQ+#x*p96s$ zH5oa+Q&e_4oq#$fE0eTbx_X73e~01~`&+PNnO0TmTKn^XB?YdoxH`#Cd&(qZGc40% zk;ky7^BcPkq(0*%xN+x;9njQxlxAQh#@fmHP-E6WG70%tB5r6OIC=>l7`4#6cdargt#2rXQSkLYR` z$0zsaN}YGgKb2=(DD!#n7JM#|(*i6ADxQ0e9QKg~|EWB~rgach!`(A`s`2PFUJ!Go z{z-E1e`JceildVG?x-EVtWG+R*{Pmeluq56;znx6hK-I&%X#01S#wVl$cy2vrX{Ve z^t|G`f>%MSM_!vur~A5*mljUvyI(kKsFkH6Mj)R2mw8t3NWuEt$cx;cu9)3(Ypj@B zmB0PuGcgIlr+%PTs_DJP|H_DmO!?S9xB(Fep5#9*a-he+kD`55cIRIlJn^diIf*Gu zxtODEZBIy&jJJ=nhYF!M7pM;;l_BYrd%}x!XPgPh=u%?3xi*dEY3qvCcw3r4effW) z4)f{;kb;s%_uhy=Q(whqjg>??$h! zvhcwBDX3HiqHR8<{so)bH6$M8{_`MNaAgka{HlO*y98A$kVEUok?@@-4(!d;`+pmJ zu!I-To&z!r|Ng!W1ODz3fbw5s|N9g{fdK?Nga2*tf1Bg~{U7oYYPyPa3TB$}(Dy%2 z+eNy0Hny0SbW(H22|!~iQ%dN0&$o>XMm4BV;siK86-F?FpfSEiI#x39TpWOr>E@m# zJImn?w>q92t@^g;@R3z-A~sS25o_)W*kv|AYcG%0pea7prwA!k)u07%I}Mvk?`?G2 zUdiK&fk!XM6-h_Vwq=8{Y!*~6-mv=f>ia z7S>qn-7fO<7Aaqdkol^=H@vQ3GBl7uQd)@;y7`bj3FN36zC9F zt6*`Ut3-^HiiKnV3g5DO%_D=sK%N_)@obuK;ofDC0MG7NApiSuyT~z1yKWI7l|<+y zfh~fq7l+onDP*b+k{zixNJDX)OiSH_j*vAB{*DU%&MFqj_6{MO-(-|eKLk^Z6gGV; zfWay}28fjQ09LNAk>iNP>V2(_D{rmja5Z#cxBS}07^;f6SYf&b?~A>Ckchv{?d=$! zYoZzItm*dYGMQE3B2JcAQfwnqSiLtVvC5HBz|AI&nWla<>#1qNYP1&Wxp!O3tY)*$ zDIsyH%N#(M__;#Bev`-hioB~WHD&;R6c=$i>h2_j8YJ!PE(bCyy+uv5Op!%T$tR4z zo2Cb%Z-BwIyy@jK3=pCxVww@7=M0Xnm>63AnJ*(o!MnyY_;v3EI7H~Qvr2a(vuI^Y ziu#=^hZaj$+)31olLen+n;1H&$WkhZTyn;VCppi{@H_JNspvTX)&%>Jh^iC(GJi8n zOYoiG;LNQN>fJ2nGlahH?mK)gbDG-vN7ZVKVKRba2$H8)I2JI5mtr=-8e5$1N=d$<^SkIt#(01Z_=`>bkv(y=E@_7H_Luw=O0&dQ9EEQdFnr#W*G(yyoiU?WmSZ^@wzFwF zsy6@k8RYdr04tO&zUI+zSG``QrrU|A)aY8FV^>*K3v^9V%rJ0k(a~A#*|5?93AK3p zaf8(NY-ZXIf`a`G2VaE2Oka9%CjW#yAJCeFT$^BMs^;ZQ$_KNJocS2jQ(x4HOlW|l zv-Zqzn0_lIDQ01#KKyB|A*^BA=JPb5JDT3%SS{rWmZpNZq56TH3a}Dkr1cyV5(di& z{US?Ys@{zyCRKcSM}<@lxObLL83~bWb^jP=yk4C@p1SU6)!)LusvFd)mEo_g@$a_L zvC;kuW)Etfw2(xr)hJ6)oNIg2>tF!JmkXrkv^Qkco9^F*;$bI189E*1v050QR{1;z zZNh00U4$*cNHESQ@wNgCiRLfZF%@DDZ=Qs|^(2SvYj+dAN%hW-YI5a0ujnl8vO5!^16=!;; zFjhJM1<(yS9+nedy_Q0~Gq@7Q&E&vouohgp%X$%>EDS_ar-jWA`B964-*9cx#9317 z-NoTBab9!3v~4d^v~}`NC$1nT{e?4~`!q-VhaB>XDZ~L7vP)L!QDa#(&%9|Z+?ND8 zegMQss^O$wODA<}fW}RU-kiR1oM$*D`LEUq3QbR>-@`@p)q0v=)zw?di_+DE*lHQ> z{IP2nYi(3i7I#e2V_h7jrdDx@qB*F!AH+erFi%fCoCE}BKd@Y(!_7;G*3M*;B>td~xcXiiHfu?HFE_qgN3~Bl zQVO}9QeW%9csKc%RsB@;S0;t(SABinrc*;ZcQ9z{#m1wOPNCAcUVeT}EB&?Q_e{Sl zhi}1fHkHu5c@KXo`Y@78lj1Tg83|@Yuo6zZtZbVfdrs&GuP(s)IwTG(yjYE0-?@HR z(^yPxAgl|6lUm1*58n&8GY6VSwYUq1!Z9`Z4V-!C2?K7b+Q|=#+p1S`F7n#-Cgu!3 z)ii$9Y&uwMTED1ea8W^;WJ#BQK+EQOg=e_G?h(?X!ylDNVVsW|Spg8FZR5yNHQa62 z`+Oz$-r*lenZ}Q6+$+kWoCEoA$Ts8_+LWHg+uiB&ecTp4_dMInaaZb5=SPF5$d@v7 z9nxw8BFkFZz+mR%mmai7AcNQnUzVb;hUXQJoiz5sx|=gG4tJ7Sr=xiGVdPVoTH?~8aP*z2cGsbRpq>0DbgB_vk)SuDfS}f<~1K}YpzXd3Z{NRzS2|*h{{#-&8 z3-coBTVgs+3=o{zv`V#jFoHhHx41obRsk+$9UY^kQTu=mi_3ftYw>voE**iusd}VC z%|8j>&#}CigQsz=qLBB@2uPac1>%v$#+9kn%Ni-CXL}Wf)2PUdN{02MXe;o=e+Ws{ z+?Z0&+ekST-wD(Ky?dX(2>N(L-aMU?o{3q;wG149dQ{`Gb&!Qo1qtthc&7U@%8Ao! zcFsW1#_EI!S(<@M@!P%R_wlW!=4*GpPXfTT)sO(rLoggO4@vlG-_*Xk{+-*Vj%$it z4wh$(<J!$^Txu8QJd!|K2wEtYOq<%xAi>3@a9Y3t*xI5z3iiY86x{`VW&;m}qpf zHr?x?V_Gv{03mYm4|m*l#uB&*{_Hxf3F&`B_2X-IEdAAYhtYTXBEn~so@r<-#UStF zSle3bk6p^3qVz9_+A;e%&x|FYWJZEclKnpBGVgG_0W z^t;K7728pmIY1EJwE(p6C~KfP(J<#+nUXpN2}jk+C(48_&iG0_CnwZPnrLC>scS6d zk5enc3-)1JEv5cs5E6@Bfl9agRqjRxMGNeOd=b{!0~x-Cu&pP@LwmYt)tX}s9R)L)36@((A`3{7g3 zPthPtIJVxJ017f5I?igX$2#zEN`k9SN(IlCwTiAbZriR5^7K|U3kC&dut&~An{ye5 zKB(t=4OcmW=8z6W%Q82;g;Lz*tPJp>Ku;MfuH7(7+NY&F=a7uAzzf4@OK`-`ZsM^s z%b=}JbS%oyCHI&2)|M8+T_4LRwlLKF2`PJ<%NgG&nZuMqdPreu5j2dQf{_SH@_>zC zBXMkT7r+ZuD7{S8B;is^9p~z`{Kk)1`fReRSf41Ay<$+w9U@Ehv@Hg_PRm7^D&Q`i z>j|QTtH=LFpu`AY?Ym^>O<%Ldt}1Ztm}c*+XA8@G?J5pv9HgWh+^7(x}be~z4O z#`~L9I@ZTsNPBu-uIV6~wsaZ`X~1TYQ7zsBPm5!6NB%o)1B%PQb)^?g9+zi>uB0}B zG^8v{)v<469B~zYGo1aR^T>Lcol*Q(`(1a(M*S|K!stn4eh<+1(uK2m}mtX{+kyd>wR~+6`jb=3iOM#T@$gP$ymT&baQE zd--GWWeg{pM3oowLxVr>O=?+go+lVGjidx6@5JMgALhd3r7_-Db7Om5G}~e8t<`kN z&7Yk_uFlF(j!upO24aAKq>F&y7a)e>{-#)2MeXFWk{ zNW88wc69z3SHPK&T0Jn3UZ3LMr`=bL+J9*JT04Dz@W-89{{+_1AEZOMZ??>TxLpzb z>KKM~BI!^;SOXD^WcFvQ`$?|@1pCssgKKy|$HB*Tc^8+`$hJYa~>}|APU4NB%L>JyNM2Z{!d6mw| zKmXnq)Ld>IA+jxV>}IHfTf>&_{7q%W1lBM_mg8jA{tw+j10>*rUbuOMduQ(xr8*9r zvA+?!t(OH0r3fZ{i%&6*es^bUg@9vG`uXFvTVm_Kd*ISod@b}xJCo>{V3b3X0inM!|`_F5xFHXlDbAQ%jBz0j*dy+mIYpI zJ&JKJS$huiEvKMZx#F9Vh4a2hhj2T6LALj`U^8*1ili4%3JVccqP6T#ZDg$D?uK>B z-84I<^Y4WQLKA%~ypV4eG~W`R>>^w7PIUFq><@CVq=k(Z5J#_KR@M4z+UH~mkuk(8 zvUkK{NM@!Mv5LypendU!G_ zpC0-Tvj+|FA@DP@iW|&+^1p)$pJJ6quD~k>u!rdH7fW#3{g0bTD;B^?m#^c2N7w9V z`?*j5aWC*c9)S$~boB9m@4fktC;b2ZyA1Gf(`x)+esR1dHk$3`rACL(yb8MRj~cRob8T zL=ox~{RZO=Lq|C*nYIiIm?M)P67}3HN;0MHR>l=pmK3dZVpi7Tf)Kja307hlv|8n` zQ)_LGYweXMnDoHa9y>5O>;kyjbNm0XWB(r@+RP}xAQom=RnAidG~cgk;0pyIufdTe zCri$Z?I}vpdO4Ox`#iE#7V3FS=dqQjG+tF|FAnp|P;#M1%nC@pyiM;O{owQEFT|(S zzYupyAmT;C{l$yhAY9N(e6Cq`6E4I{quW98ST61x!eVaO=@RUIN-{ZMkk$Q{zCnWX zXCcdQqB+fox)W}#PWIEwfBe~b9GxT?$3KlTzszm0#6_BFucFS&4@u&ZfxQgxtdx2~ zFm>^@*InMaILeYPZ{K+m>8yIaof|bN;{^{!do0W|>0O?EMgQctWwN!WPFM+o4GFTN zopToV_i+dPUpH>}15wF}$4zaTAj-t-1hd3u_xlA;5MygFpMl%xAJM^G7Vp9N&Dbdc zUb@7FFjD@oCWRCwjJBus&njQ7gUXpR$?Sej3yNL`%_Rqa&P`#XXIItn`h#hK&|tYo z>wg<5TWD3nc5PVr7eaCaVSGRDqB_^APQN;8)XG6`f+G(U6O+LSiJ)k=iMv(iBYGOI z5Xz5y2FWSAno%%SeTtYAiTgfgCO*UM*dg&nRwUs(^nhtWi!D!qV{*plW`V(;{F#3^ zIV&o_FayQXo~so<#dcZ_188MVn__>0qjrC737?>M##;T&c3s7yKf6wVq$_Zp&anH6 zC)qq+ZjD~qxL_nmpMeA2lGF(+NuQ*80-xE$di73^@!$_G8?Jq@X3&cK>|-B^dSAH> zE+7B%H>B3fo)%nld|UO+6f^;>Rih5Rbi|>?em~~vR!@-L z@T~qq%<<7QL;c?1Yr#*!$_GQsDS)nJRAV8hW@LP^9hxo{IkbnKtaEC3vaOWN6-_Kg zh`jJa6h{MifIIQzb2%>eU*z}TeHt9S?DiLeNX2}cxi{#onDoG!>MumN!e0nEtB-qu zM&)C#)BA6|s~aD{qdX-Rv^0xJe7>xtk<>x{>#rTvNSi(uDG)_^6`=Eu&~NbE?4Ijt zu7(und19@u*zWhssXN=aTi=~o^@A<>KZpwyXZDLPN_O@;6=-k@J{M%0nzbVtNukkB z52V2ElNkhS4MnS&jwb=;>(_d0Y^QJ$Po9SYjkbP@-e+$==_hnbIoc<09ub{#QSaaj z6s%DvPkKYZ@jDI?0kFr$pbs)``7VCx-gy!qVd9GOP&3E!eza2K3k%&u@!AGsYFbF9 zx{bF5w2>kHg=qRs^>NY`d&k!Qt)VNw5;y$mNy7}e_eb@<(0 z6z4C5#k9=(f&<>MaqzS#)=&Jl8NZkku}oo#$3W0~m%kymH>4rfE@^t>IUe7*U~HDm zZT|FO$Y)ErIoFh80TUV=)X6wY+Lw)JvN**7^FbWzr+&g##4s}@Y75#T8f2X4UK_GP zBQRQRlqh+{Gt=V%5sz{MJgtR3NOj9#CT?;}Y(*{XsC>u8T}b%#t0g|F9JPV!SK@|- zfwkxs@^Xt1G;*otKOsaSs5ARXu9+!TvJ;FsyiO%lt5v1Zs~1@lqBK@beW&_P&6);k zW^$`1wIV4o#%7)x3m%X94orFqUtdK{4UEH;?d|{2e<@y}6hcs-N*zBdFuTOikthQT z+ENiZyVeutnYa|U@yUn`cf{Okj>TI-VyYl;i1+*a{(p8eb+<-4pY~%ezHzpP5T{hL zmcZv{`s)3;ZX^x^$v7p9-eZl%XZ+bFRTu~RlWf?jcsP^d8sE z0*++4_j^!l{{Fr)5r%`S-Bhaz4ED2~^>}(<(<(TG#v%wA4m4JHYBU&C+=M@3EF38Vj}u|KR_v0O*$MS?_dT zF3Z7{x8IuW!xnzy=C*5A2`X8ny}Wg`fr?$D6^4L^JfcpKP}XmzR1;6WKG@=dA>Rl; zWeJ96l}d8Y{D0mh&2L1@9<|1mB80pK`&+S87hXk(B_;SgX%d5D@c~vxHKJ+-O(nQc z5UNjTBb_*o?eMw#8PsRActUHt2b7^jFL3LDjHLX{%0K?C$Mf+cKV(d^{GIryHt^#o zzRFc6czilS+`IN;+iS0zp>K|a}x5P^FqHXr*ouvkN=4{?^IKJsWdz&#H($JVXl3yoSgt=>stCUKFbxo*P$_Gw}?av+lf;k?yhJ4RDX}4)TE8lv~ znHG$sM!ZsF*8Q5q;=)j~x>2Ibi$S2FL%bO$!jqG+2K^lfKrdS0fe$c(_ zSRi6JSnvn@(kUf`9=jI{S}Z`*2vb6piF@7}gjvr8`BK?Zw{hdiASeg@E-!?e6>dxg zqK>hg&eq^vr8w3XVF$RIKk>jkd|5Zuj1K{P!N!e89pBoF6_a?2^$J?WiPFJbQB0K2 z1gL}tPfz=H#(7O5ejh|Q=H&F;#x;blFkJ0G2Pp2jGZ_P7jLb>@Z>~Z#b6Y<#!Ye!U zF;E9%0a6IMNcg5!TJm1>4;lI0u~;h1Cwv zT^KAiean7RjSEHyDu_hu?oh0F$YF;9{ixoA1L!li$cGf}rJnZV9*u zFXNEeC`rug9Fkeog{14FK;!>!BFj+kD?9rC5OZbwlhUK! z{UGvb!pa=|hIO33cOfHr?2l(PA2RDmTHn#otonhz{E;Es?ti&i2g_aXbT{TTSFEFoGixSw~};7e93 z$P>D0QS%of$<&JoP8HH9I+C@n*|Yb@x<5?U_D zA!`D(Yl~z1?*S8(Ysc^SvyVYj!;0ag^Q(GYLV@%~>gSje%4X}Xvnu`=rHz`AGmCX$ z@oWy72r>Hummo`>L{8#ZE9h}{>zhxi&8z&V;Mo4i+sNhTGFx5XIg!Or;FEq1;VoWw z+t8*Wpw}%r{USq(2>3J!a}ulgEvZWK+AmI2OG)U;jV!b~0#=NNd~pc&TRkm0y#OpW z;b=JG${a4P=x2_KqlP}ppID;8uP?s~cQpdU1J6va>t9)2#V&KLe|~xE$%y~Vqd<6? zfgOKBiQZ*LV)_NNH=iT}{}Y7{0w{0=+ z?vn7oM2WK8{I&PAUY+wG6~-QSzu&~$l0=$2T_QRDpfa`NPY8;=&x&^lI=|$1E~tlA zO`3dIXZgWN7F{x9%e6y|((On6*Phd6N6rH`7`vnW&sLq5#&)M8rq$B7UwJmdU8bCT z4uih^8B;XifV#I$scuDHi3u+(i@7%gO{r;Rz*OB#^K2vxa?jLv9I$G3WIs@Z!5nA+ z%&!5{t;xiE43q>RfcPy^HWUQvR3U7H=HQ0{5tBsc5Fs8?;;*S+Fxi!4fqxTNc|D%Zl zOeJY8{$o13Be4Uy`7cBcnK5aSf-LKMj>7Oa@H4T_@4`Qjz&{-41`8OtWqPoA@Ywk@ zR_FCrW*4yJH2#6UDjJ@W!D1JgwShmw)Eok)O!Cy?Ct?q8o3tqZ(~o1YjZwks#nbs_XKNviTLAUE(XM=bdsm z#lIm;`5+Y?VuSf8=ukGObB$grlG}yp+aq~jh=S! zEm-*V-^~C2rM&)k<=P&glU^&RyrLpyr}ELUcCQk8rp-0XpcnAV=34vM->Fld(pB(P zXO^Qk$9lVCLL?=TyLLaZ4h&Dj+U~u#WMz z^lXDh${NMVr)vJEOngSy2Pc`!)A2L1Qa|#SofBehybwuaAx4UEI!os#gO-h!PEHjz zn)CiBRGJtR-B!i!a82?o-9m7jXd!ZhqWAXnh5POed&!>SxP0j^gl4vDEqLJS=M|B` z_8RSkL|eCZvnOMea-KN!zF#){OqfMKr6)p^!LxNu2x1_qc}Rb!@9dEt_l2yq~HW*GyOF~h+sl4btGoD16en2#D? z(vstOBfI+6c@&W#GRqlJdBb2T%`c2qTTR7VZA}JzmcJK(7zuGdzoEY*hE9hc8Oi4rcq1{6jf z-_DazKQ)~@hB#4Exp)qVm~ut$Qq0qJRAJTA(z*H`ED*%0%Rtlji8H0R7zwvJcjSBHyVaqQ4hJ^O9-2)GFpD0v%B3OGmTR==aoL3 zr3Yvehx#VO?%V4potg;-(Ba}XnyPDsfP#}b0&ad(LaY+GxR6h6AN!*kt&i8WKCc*3 zz41J`3=8+#-wxKjEkb$Bn5+)zA{3a>Q4B*2#wMFoA!A5(ITt3P}U0 z%dBl>zh^a5o}1oWXMqP5|1kv;tauM;{j9b!PP=;}tF-)9UwrBlVCS+tG|uFvdquGo z!-?nA&u*{`b2w%xIE_!+#D6^NZQ~Zl@wEz#&x_$LsjC z#&>Q;A4OLZiAsy@fA0SJ;rsfYm=(5&pUWJC{$!Oee<7UTMaWu@pG3&?cMh?h>z4$M zV~B`OpyU=NV+BEVdcMS}!@0blAXvu5n_gDQVQuVXv8C>7oS5Rn0A4E*3Zr9qcy!5w z>Xf4(cbyaI(U7l1|JLiC7y7?U+FAd*SlET#pAk==g2Iq4ZHoHhg=-(1yz9gwaK~Zs zZh04sn8|qqTPi_o*Qx@c4^dt7N`?ognOCZ8mxDi?fT+Mz5$s4)I9Cb$Nm{P2i}JQw z3KS_S0>|#86%fHKZM8sN^ZJx*To>8B!EiPC8pyIFk+fF`hA!U+3t!5zSe)*HS%5UJR zPaK*$LpN6`tS1E-JVRZ4RD7_oznAcd8R41%oo-;8odr7(+h^LD!1DItwOGIR$TC5D z!C1+aSE$tP`i=WtyE$>dIP@%V;{NR3n}-ZBVD?gzNqdC_UJ0?%0#}VVyeO(v!fo6q6%UK_ptPq z`f`?c|8=j7>y5y??mV?ZF=rEV#>I^{jM|qAzd(M(1;{hI;yn(oB^M;#tNv<2VKx+A zWt>#mrQSYt>8vqq`r4$^?zkAimdQ-iP{*ZvTNn`j;y{1pp|RM6*HWFGy0cTf;onv* zzr7oAe_bT>n^@f_!fR~=BgC8yY=ko=O}JvA-8YD*WO~zKm=82Q*#Qi@Db;3k6NqT5 zwFYv?Nr6@QCU||| zXBc_Wmdax^7Sc^yddYE4ZaQ}MncQeZgg!zEsqVrq?daC~vSW>u$RPc8F*@g!u*&1C zBI)%Zs$k}x|J}{tGX3LR8)#IJud!e(b7shAEVFp`QH@y`Id}goZ)CfAAT}$idNI3% zGO2fyWul6_5G~#}IzsuJzZj;)mb71FfhbpD8Ww?um?uB1BuVy7YwP?=R+JDxBvX7vv9MrSYcM}J4ZdL*Z{9$aWRsV{#IY-)K5XyyIMub%etOBB(6 zT8A%>Xh}LA=IomcJ8O1GUOGt53?hXwGcSX5_Dc$b{iPi>2uph?wE4j^GU07mA)X2k z{cu;^oGKd0OlNh_T8Q~iPY0?=Xj%YqcCwvu@gEP?4|MdS$Bk)dt2b6Y$9oY)kOMnz0~{B$yOOZX{3vGc%?^K7U=Z8H#2cUl5Cpi{6vQ&=-Mue+ z*cmGY>Z6K01^$Ila#UM14yUuM#v)Fxjh&;|{pB~uIWscT2GPkY!RDM_znOq}r61f% z|JSGaBdPUo(jw!QY4)Udo)u7 znO{M#6KZPxeW0k`K-1#fdTZjsI{u`+bEp-wcYa_#@BcK`fKUm2PObH4c5`49;&GkJ zctMVTpd>t9@xuLcULr64)qZ&=nbR5wCU{@*I~IXu4IQs9->CMq^~a`;K%>5A5$pa`o`a#12vGBfRUXvIpH)YY|1HfUJMcLX&AMLKs)hjQtps3(o-ZIXoI_WY&I7PuAe<#zEvz3xYn z*_LbW(b+~hoO-;aN(qFJKvi8?iZ0~uq;%4aX&i(O z5%9dAvxiOs%O`Ui20TNXi%pYR2cDKQJWnaVMf!Q&{&Wz~M>;QPJ&N;&o=g( z^&UA$66NBF8I^ECC2IeZHz zmyB9evQ*c!oRJCrB{7?_pX;cO@>x{`ND7l z4k$I!ME|+r*7~x4*i7$p zQ+CQ*FK1x5+1-T}3m9^yRd7-M&$3GNv@;nON@P@?JTXLE;}sRQCuaM2^S{eKWjc@+ zUy0(aDK)Mz!(3toskDF^@uz!8$5J{Ke^SY8@v5_9O7BzeA$!q^AT%A4U<*r{l_w;% z_+adw!z>P_1Q}{<2gZGPoln?@J4{#qCYHfTlmB7zqtghjf%-^o7WyhA!J3lVjg$n) zzfOVSYhzll5f+7=q%W8dFhFUJ_S4bqx<8b2%VA=X#m)$$_Y#JH)i@c=>=Zve{-e<;pn8{Q)LYn$yjCm|% zID^C4ey2m12&saSP&jq;3Vzj82V#%HPty0khR)J$60t;LSp7Po#Je4HAeM9NWc*L@ zMPE$!Vn}*4D2g3f-;=@&(K~}46i-hyZC9%hd|@uo9RGVX^Z#5CG@?bUe292{Q4>`?jFcOTlafQC^=vC5$aR@26zI)`>1|)rf32NAnF%$G3L;?I0*Akpxf+ss6oD@(#0==$hJA#XRy>x_}q0)5Wk+tX-SO?P9?g zn}#F(y3n;CYM_VJ-kdPBw@#JdOe;H%pGsjWOO4GOia$J~veA^*pC;tJn*zyUIy@HA zy{6dYEHd(eM&CD(?xuFm!8ytdm@2$lR^R%Y(sI%p(>?o{NM}o=zcZ9%iV2H!aKu0n zQDzFjUbhK{oHJ|yZ*8#;L&`k3R6UD!AAi}t;@?sj`=I+F=gci|eY(+`x>{?smgi9D zT2jay*-wB*tQ$8u9SvqC2u~bmc7+7-yK?SLljI4q&fnaT!Amxm%SU&fz)oow;}G%K zx%6p-4izf%O6YkM+BO+^kEHmiV~GKTodP&<_z5os*jR_~8pp^FKW$6Ke4pJB4>%`t z>{ke>CEU_`0N`22PtHkA#3q7=%keo)?Z11j z@Rk*;lU=liz!ctb4y+^B*{qT#hb=@%`@$A<_?|0CMX@p{@hKxl+m|kY^{sP_fiT^Q zT|Oke4e|iERnm|B=o&45f{otn=N}P;I5bF?OAceV95~bjr);ZhijXH15Tp18O}KjJ zF1*UwLSNdy@8V$fo!PaPwM-`-qPX@GaB3B6mJ(ng>{Tu%Yq8W=f;4Vd=&_Elt)~Px z&9&N-$&ZdMYP!zQ<=be$@~SpQW|7cmg?_ifOAHWkHbH%Ms>?stqG@ z-pYLr-Ja@aa^#Y^y@0UIHpQq%Uh+dh^^y6u5&Ms_7iW?7JcU(Jn*zH;-|8@q>) zTT#-bo}bC&2Zfxnsl1p`HT0b(GsNshEmC_*+q_cxN!l;rB8iRNJGI{H27998&OMLj zKCc^ZNU<`lEGb;Dt(M5%u((g3yQ)&h?WtMmj$z=m+T9jKA>%-0l9 zr|#ZlDw5LXhG91W74tf*_tw-bCu-s!7(9yJkT<^srxCtUib{EmH8kGY-_nwvzh82@B=5H*%1PPsjhhkE4Ip5117RWVi)doJ* zR7aA6?CSeYmJYC_-MKQN2typ9_&ys~%A^mGJh82FEM)7Ob|+9#;}|*H5C`oh*y# zT2%x{dm=2xxX6Eh#~-&7izbR!DwQg^nGd19`3x8qmRm`2sMjSj&^ydc1miPc(v@2( z0{&F>P4O)gdy-TrG^|bTGm-GmJmq3f6cf*$vpAdA`2n3;{*7n7dhAT6~?p=7nFG{X=hRF}tk z0QSDR)lh3%V8!!g_<-Dk{87S&C^wz}*H0+Ydz%*5Hu&>Q5~YY0Wh}@zns$@UA-UJ| z5Wk1`UdYk@zBpo>G5$gNwtW_5B*`sj-D`alGqJFw;iQlM+ctpz4?tvx0YURo&Rz9_3E9OzISL z@VjH5{tAXkH{VJ(4pL}uG;e|OjKmGQvxPtEH?C;JcP*o(2|KTE4Bd{T_cFZh>1+?D zs!ad=^n8#!+%L4v_l5fG4mv9uy;1_+$->UK3~tbZncPbj2+eCvyU`RjhGpWUw3})7 z%1~=+7m-)3SYdQ3X)k^?&DhpL{)PB&2MFrDPsl&|qGr2v8rUQ)HBQ|G_$_ZnkhF!I zb|B`ML1I=D2r29O-BZ2k(12OqXdte!$d)=g(~iI5ERD=JH@UX8Et0cu?`Q`823a}D7r5^D*E%v{tlbfDI zg}`Jsd2FS1V;s|8MoMm4wmr-U(r6Lm0ECrT{Bjykuo5vty{XZRA=GA6+X{wdF-!!u zAzMGpO4yuiao5i)9GK3QeRnydsd=T!Uip{AF@?=IJ*pMSQjl8hO)%h~K;Ksyy!LOX z*Ng6}PKrM3{|+rEcc{}9bY7Lr=wX)Fn_G!E!M#Ytwp*MryDd{5_C>)O-)mx4U{V5P zQ?zQ1W;szG9J^F3WV?K++XKRmQGr?wcRbdq6&b#SMz?=EU#S@y?uip3+*~$nv_E3_ z)U*oj7xkb};v~3CD&WqPdTsDu+s-YlbO5gZwWS;ga(8SA^N`f;?#)X$>G6jX4 zj1qlGGwT+6ko#u#zM{O6Bu@4~@(U66jhN-H>s#a{DbLeN1^-$VliJM4+wEKB4{hn3f0Y#&YfartMuTB=wy5 zJVP#HuV+qh;AYk3wa$Ys5HftkWmjjyTfEds?I5XtO`Z(((JLh;%}1c(6~4ps;*(*0 zCfym)rc=YKjTqi}^IOVMF)6c8WqvDHRfAe>NO0=td`)GJmF^%PmY9Q{Ty{TgvVj*P z=cPQ3zP&FoqsEcx4UssO5@tMXCu#M=g#mwW6NNdrV-lQ*=3SrXruqbg$R}SB;8~~6_*mZShKc!qE76_N1-P-Un9ySzYczL+sNDKV7LC$* zbvZ#rMPLwf&VP1pTs5eG?=Q+qK!2IKmOUzFenh z>fK_w&M=L#ys9LM7N{sJ_N#c*pA$j4%485pNocNjoxGNFkd#!a2fgu+@f2EDDNXn2 zZEFX;FRzc0=i_TLqhzdqA(#dJfnc_pEd6^|Afs7?KnDzQW#W5E#FW8Q#f^0u zktzp@5<0yOttYLYW#oL5IM-{wW+Rf15Y?*r+D&GN{a-5^^KqvT?3IWzwZ9g3A-L|~ zH|1U6lyP%Fi-bL%EC>iav`jHkPff8im#Q^qO6no^3@$`<+`5OZxv`WJmR@G5`SO}x z)INEL+uDyJS!=%YKk2?t5i0 z~`}ryVWM;(HOO;sK-Y zE@Crs4TXFG`2_e>pSe(OJKso03~`~w7T(5R$kEgjYpu23mVj!W!)FPI-N|Ss9bbL? z20b|*pbLm$M5j^tEY))AOW3B-YHg;Wxx1Yi@20+esomAaHHr!_#XzcB@K^Mab&C`G zC_XgGK1kma^*mq%ftiU!hRR9}@XV;jSBpu{@hg6_Twdpp$X;(;ef?x6aK~lF^f|$) zo)D+=`HY~)QjF3x-f+6}3m9WS9nEgV5^AnXog#X&b6;)CmP@Ycrm1CHYh!k@?)ptD zW7Ei-)VHDbHkFu*ilg&YzSA30opzjcUHIT8K7&$oZKjZM!@84;B6Ln|{B$doVseec z&cZ>w7=W(CUAy|Tkywa-#Q}a386|F&no7BYw>HJ*m_EULw=e}V)= z#c=&J?PBwHb|mSti;$ULdtUr(7P* zo{brD4&rmMgP9e%ICv(o;zF)S3287g@)+XB_D5YTxig_Whxf_j&zNaXjz?K%Wn)WL z4KDA|zZ~F{h~k{SHV8kaf6sD7saU48lz|y!_YfjaYkmCZOoiE;4u)x#bM7)-z3BK%pEH7kE15x)4gTi}&3;TbFqf-~6vHQk} zaR>^tBzqheCM%D>iiZQ$fvCfRrSf z<~o;&N?Lll3g4KOVkyWM*weUXw+-t7i_z8=`#WY>wkI$OxD)CnciOA4rTs@SOpIZGWId(=qu%#Sphut(c7WZkDk8k z!b$?-g1IIM;25xxM{vJYCEIvq)ztVK17+yDyKo#O&FwZi>^`asf&VT3*)bLyGWL83 zH50$IM4#>qFwNM_a$%O?0Z!jy6mhOvbLuijNLw+O>bd}(68hkK=P(jhQT#3j&GV~X zjxx()V^x)}dy4BL3exQ_dqX!icJo}2e_{P>6D|6ym~8 zFayQKl5FfuOU1K`&@oZ{FS_0_IuhvX(@n>=ZQHilNjkP|+Z`tr+jdg1ZQJQM9jiM| z`qqEmcjm5{x%X3juC-2`+WYL^ejW-8YNn6q!<%Hs&5SO)+8sCSNz&OaEyDwiKeMM6 z`Py4B9cl?PyY`AGMh})sBb|QOW`&D%%&bwe>{x0X2_=f6Q&6(nQqZXZ#NNA0@jDzv zoM@6^IJ6zsz}Di)Qmou>8=9|U$`Q%XJD+8uJ>HE+p&4%Zu0Gro$>{euR~pWorCGm+ zV5!f$w5ms9dF+WQ6moj(yFD=F-yfbcHg0xC{QZ2N<^lTok0chen-I0bk8ykO@rx<- zc*g;DNhh7jtcMV5O2^zs=l~|P-UW|k-wpnGTn_h$Y1PqG^lya(%QXDh+&G}?d@>r& zWMHJ-2+DkLf?2V7M4MI06tYBdoJ~$k8NEp+oT$}&ZMwR<0+!4)W;-z9(f4dSS}W1W zFcTjwwm{`!EhTu(6mJd#M(;H9h+6wd*W5eXyhql1CC!VigR}Hf3d<1>Dz5`Ib_j`} z;Tlu=u2ALjc8{c)!A=C%zx$){G%k=-AhxMnru-cOZ3D4@d}_j2znw7dsN|7nE>RIL zIAfe-5-ygFp0YayX1h>#dKJG&i&-Bi<)NqDJK<*#eMa|x+yW&ckv9eQhK)9DgNhud zY)PhVooCE4lpV)}ld%A`^#QW`@%!Qk`O!)8jRERu?7lzJTdK_#zbHfJFr=eY1y&F| z|3A}`pzI{#o+rhuo-4uWN9MdPSE5eXe#lbBP*aNiDFodVaJzE9nwoo9y%>PFi5SVv zMee=26Z-qgU!a`zX=Hn(iZ+)XO|CYq!nfn$a*HT;PZ6$;z@Xk6t&eZRX2cI6B-w5u zE)pk4{J~HR0Gl{qKDvDz*CYf*6Q*skTU2&{w@Dx@Xp%h>hRb?ECyTfg0xWn^BL4@r zUiJ@c44LoOuCTvdp2J7SU$2K*lzk9`s-LoUadmB4qE$irE|4W_4iFMGiN|c0k0(Df za)ZIrd%-4(YOMWEDi-Hg_-m))mJ)hC?XR6}iC9S}({JrM(m0vyXYP_DHdx~^Fg!al| zg7THa4thV!t&xE+{2CTmO}sGro~hT2X!bKszSPff8+V))SsN*Z!k%e{Mg2MHk9b z#V*(9Oj6}nfy~N3uud8)IuO3zY5%iNl_jKJMV1JZKwejFli)gIlpH3pa1d>}x9|Cd zNqzX?#~TCq#+D-$Z5D$5s$)Yc2QEqu6-<&KR~jy3Xb|-Zc5VS@09NqV?$V#CZrTcu1`G&p+9FFcDb%*K-*Hp(w>0ABBp%njopd3 zi-#?8)Lb3t00hg8Pq9j6!so$_PZf)%Jt`^#C$b&v4xy3Baem|b%=_gdcZFj_FwVNH zq%q*_C|Qq+~C1xIDAoO$eftH*1w=~}*Kp@`^@Q2RrqikD=JkuNWgxLQ|+*SZIH zcFC9@g~BFCmDXhjZh+xds zM@>6G4)4UMH9>*=5%*$vR0&>Jnc2sTZJe44Vwf#s0*G{S=PUW*uOU!aCY(5Hc=Bbb z`#Eqj9BB8M#BAPmKWnpurg{?H{{ywzMexplVquD3^F9tC-=u&-Q3=Cxs*)#AC)XK5 zO*uv5$hkO4cZ^oqNi0y_r=j_}D8<%set$;~IV&v;ui)^t4cd$e4p%IfjdC?DBtLXQ z2ijdj2IQ-AC!(Lm0@)Ulr`&4%28a z;I|{Lvg?ENO7wWr)vl*cjWeji3m<2W<}d3DayN?cpVN1{kgL*?)>pS9lvj{(gG6;GN7Dm##i$E=Y=!R zpZ^}ko_gi9pQ^OuuG=(ga;$#ZuVcTs;m;)=Eg(NG5av)%TK*b_<4+_JHW?F|S*Yw7 zvxrOdKIY|?h+<^5m`(O|&Z$Z36+(RtPv2i^*>))Xv%Cn}>oRnB6Y=ll0YC5c_la}} zwBHd#3HQv<=Q`{t;XJ18qxw<*4JO%)51xx*!s@3~(Gum&L{^Lw0S5`Oca8#Ok)tJL zOei5QCF7_eq9R+g(F5KC>C~r!B_T&0|E~y_<9B9dmm0wnt}M$_pjzg!G#?Lbo@fbM zW(O?gAqEu{7a3LXL<|wD_QajbfIS4eF}wDc=IaUp?_PHwD-+JE=_SocJ;pS_YxRds z2YSEy-{8afIT~A$%L<6ArANK1tyQ(^x65LAGAYLD$+B-7kQ%D=2J%&=3PwO?MmdGr zKij75g{@V)x<;<@YH7f01a}%I3cbsU4iqO-RR{oT))&K2%Af+pxUIO&P z)^+xg07JtDiy4`j^3O=CE55j9eMQM4Xr$8O5y)QL(_sA-DqBP z$z7qB$QfYaod7%+WGKc&D?I;FMjcb;ABoY)(-5aGqFIu`);ZZ69OO6%QOKN9IFwmZ z1Kh5F@{Gr;?!&_?h+~D;AVAg`>%qQ_`l3@RvCoy6$MAXn>b{~B9!>fYdNQx0Uqc+V zC@xrvnRj>znf$Q%0v`#*JRv3&RisRm0ddl33`!6|a)dch0!bLD$^{4{WIEQ`A1{w1z$+1NoH)gg?OhX;>3GRnGs~{qe>y zh!NcI%4(B~V9NDKRG&RWFsG=SC~*PVm!*K>_fpu!7O4>g8CG!NCbbbIemCAi7)fU( zMzs+zeA*5P7pmz=_cjQ8_Lh7Mgej>olOeExCaLRa>U(U<$aczA8c)*Za7Nc`WcwyR zH0LhO8in84Q=9Teo{+hFHE{pHbaLgMn?i!1q@0BWayz>Tl<=1fROSVYSmFq?Wg0bw zSoi}l5_pYgMt(kNk*)@(H$+#Nkv$2LUl+5$*4X7l0z0Qiw9yS9BRemIC1HX{P?UPZ z)e2<(g0XxFK@5~`-*W9ur~2NDbsdx-!5E&8Ug7tJTP>mJv3Kg-*zO+v;ifc2$s>I78M7zdH4Pzd&J&3RA0^#eOp2*iNag5EML1JvGJrG=7M$Nb!e z9Qdg+)L>T2or6!nAM(=>XyE2_UgT4W+Bx9h;C#rdA7RYGdLqL-N0#{?vCS|i0#J8c zbfCKw%lm=g_K3;@(+T?EI$#)JADPX*!1GF4n!`@&|C2S*tSAC*%mH-(liC6)Eh9Sv zz}Y!#*ul++3wCtf7H^)Yyq458^ZvJf?rS_A`Gfch=P~`P;{RB_MH+mY+7k^(vOa~{ z89;UI*d**jmJq*lz1_mrg5wXVg>K=;RS~h9aweb!K6_^JfIX z!aB1gu8Cx{EnEfO*`2mu|LiiBWGf#zw*HBTM7|E_MCRiAdE8S*4}&tUW@>F7*UkHl zB8-fJenWpj;Ku+7F>3Y0o%u{(Yz0J?(XY%f4}tsgzZDPvz?OsN7ziJo@*zN*MSQO| zGac^W7;O2tVwXj%nJqu{RU20pe#C|>HZjliiiIlYa@v=y=CEsUS9N%@Mr4cZmLlbTqyd(&`O?0B)7D zosAS%?)$>*M(lx~JzjEZakJS%>?LQIw)%>D-$@h4?i=~*3*T2#0l2wxzGfs%^ zFK=0PKkfGtM;?uA-amqgp}Tsgyvn=`va#y*B*NLSQm!I8YRMxSqiU=3W{Us=Z4s#4L=8-l^MpRSo^RSGQN*3%7%(gF$?O9!4q8 z<-J0Ge2nT6NCOLM_ZuIV9==veGi zSss`r`8dr|^FPa0*$mk;av47$d5JX}NCiojTv8T+@CMnP%p)5c4Lqr_pLl%By2$ z5T&Gh&>>F=Pi6szvdGk=VUs*dpbKMk2(nIm#z1My`p(P0!X|BcA3+iSz%Cjfl0M(o zt%A(PA_Gyl5niVso_nP|iRW8{@W%)&?4jm303@Ei1cK2TTapkQ1>y$O)ZcH&-`BMK z4@u?z26!WtJBUtm*SzP8ugf<-%RNzltl^Ak9a@bS1nED1&^rw@7yQ4QgmH7b-Doze zd-3MiO|QWqCz|M9NQ7vyVO!>1{n@e?eQcaiM&IA`+nnLi9^lzoEt2hN0k?RV%wl#! zz>0~a(f?krH@9avjVueje;NlJQ2grr2c~uXDS!Mqu(m#4&{H+;>|A?IH0zx}6YrML z-y5AGtd6~5(ngm)%ufFxX&MWI!I(OAGEjP`KQhw(pd&Kg*`JO3f02$rd%XWE%0PI(^I6b@-2i&q@S1CO-gw3_E38MJqC6wq z2(z3`swl#aN3XN40x!+gPeBvk(ykd|Sr@A@Z_toe&mW!IeKq}%+lG4{TQ8Uvlz0nJ zdH5khaofdj`I+Qav5&9w4{WlNNhr`Wy1X}6yo|#@X_7&{HXNr#u#1ig`IL%3ai=%E zav%%0NZ@l(kxW-Z0XjoDhQk$zP;{NnWYt(tPS{fs2D+Ni`htA?#Xfjb15`i(XcdSS zuNuo930a3dKc6OTDa5jToP5D<7IQ{!6#id{Ct2`o_V%e6gwI&!+AXk)WicD4o3#K` zKRcQn8VW!frTZqk44k{n_jk0zyGQp!a^KgH;d|)y{8)L>BRpwMq>OCCSG99?OGlOj zL`%Gp9#s28N*KTUI4sEEXr0O--~?c)hAk--+>QNJ<()by=yK^F*bxlj?M;C&*Ie6| zzHj%dW#k=a&aFC+I}38MiFJj;_wSv06dc6%KY1ar2!v^p50f&Ui!KvX1V-7tO9AeZ z!z%%qO-ta+Q2dm;)29%1F4y%9Art8n-r7|3ah2m*7IDNCqzHaIOPOm+eR1p_HrVYv zUKHj|j46nB7#23BqgxsHqGlmad~)wJ4o(_S6eu$UN8uC?mO~_;+!j+U&O8perU4c^ zv({cQr)-QqIjwFS1H&yq?)dlu(b7AQC1W+_2masWv$iZdX?Rf_r~(AuoITndgVbgX zqd1tO+t%&REdeB-A&V+QPh)|ZfoHd$zK3`BPCh0 zd7X6cQ$-*Tyr(BIdpANH3pC1L)6@>x>3-B}&{BVYT_UdCg}%&BBO?Fp?=2b}C16Ja zP8Ei)V^2KoZQC=aW3wi@uC;VN9sQAK6JCKU-Nu$K1dzjIJUpU0L@LbU=0jE_c)&6k zX31(4A2H1P+nd9g+?T#~+3oeG!X9+$j<^DqN_#*M>FOXkzT0tpv8G4bel6`4OY%)P zNaUw#Isnwkw?{a8u9B%FP6Q(K>190EcKyAlc=@TYk#zjKSZs*`OvzZ$-+ROF0=@IX zeD}LiA)|hZV}}nnW8?c=Vi_~AJ=xNB$YK3KCZdH8bc_nM>m8DtvpQ4$d(`hjtyuN{`c0Tn6z<2k)SZ_8j`wjCTd>TdzMpTtm6tkP32 zR-CiERWlU&vQm~xXZKRYK>ZxNjpvs?h!Gr&H>ie|b51Gd-=EP2Z-0j+S51cCAh`WV zdlm~T|Jq1f*V`mAUQ@anVrG3BJF@ol8gl(xveY&N$M0GAj2yX_C3{@WKo^IOZ_jb7 zW%!OaVOjyx<&Bz!<|U+0m&`FbP?cDDxQSo6&gbG96mFqE zL2M(22P$(&;$&w35OkKjPNQNUyxw_#wcSMjxF>`MXTK0tynN6puVG88me4vrHmaiH zFIhr^9a}_+7V+|t(c<&LF@|A?h92WTvEA#?G>Je&->|q&A{5FIK(o(1ch|{zL(Z6S z<5aJTBDGgZC5eQ6CWT6mpMY>xY@-#@!YM4ZIn_#)KSnt$`ekH#CN(zEN|^E&CW%(a z)?a+1FUJs@ zf@^kkq<{ru@J3*o&T>2b`m%2ggR9~On}g}?6q@UT@ml?afc>WK`TI>mR+W*=MJ$ML z?(`QrU9$kB4hm3qM@hLIG3qt?glG`SOMpEU{y=K(y1#Cgc^AhM_NvO_I}Tps^k41i zLSX}czz{TLdiShmO>2)$?_ZCSH0X9Mj8y?XJ;l$o=)2u&=x7pm>)%YaDU4hzP0nCD zLgPs=T&1YhZk$y0Q6c5zN>9maH;L7Z$f1tB!ZV;SF_=i!T2NC*twwiJhPKd)J$AEb zHennH+{fBKPgG01x($(deH3h0tkw|ke{yfjJ;cuTr*`r{ND$X^3yi9L&W|DfiACWa z=@wR7I%Ja^XNa6dk}7r2;b-qrG|$&7m^wTxKh3J$x5jbUXyNUx>BXgu(E@06cXK*g z8Te{!i1PmMb7}pe$;mWJE{z+^%_5^sAdVB@T*Q`Mz@iH)h?84Db-*rJ;RG+cfz;U# zHiwOO{Xq3%bSb>-J_271N@oTJ>K+L0%!fdod#|{M##Gf={v4_CmDesyRcEpnpIke> zDO+2V)SRB8tjo`_xmwnoURL|1)V}G_ZXiE96DSN#2}!9*k1A=&f``g2LJ&Vb21JAw zhUB1>j)XSS0%=ZYxOu)dS@60FS`mCo18a)q2azj|r)HZ$bfi;WO)hQuF$x=!at#7c# z50^MNmb`u&$JQ>cLL$yvr(W&4(d@*Ype<|k`#&m&v3v&-S>{UrhV3tD_CyZJeMhg- z?bX`P@B_BaXUzu$pB1djXH(dA=dFA6%&Ind1KUBW#SE<b zL5LYv>a;Fi&W!oo5hAQ#oxQWqPBdKxW+Hf?1~=tATtXM-kgO=9!*N6>4_nkj80kN- zE%oBN&?9>D`gaad1ny(R?I5l zO6O(l*7^SAvS%i!b{!Nd_9M!D@BgYk&sg189`E~op7SI%R}gFiYQ(6Imk)bamU~N< zS3TF*Quf*7)sscz)3PV2o>8O);RQ3Nm$D^+-V~e~tS3<1&K=Q`vPX7nnkT>4*5%FCr9o5xx^A@Tz3D!{=N(a1!HS!hcy_jZuarnlRC&ZuGa9rOTKPR3)HDpOKSQQ<#|m zxo9m~+!r|qOJ9?Cin0vZf&KM7LpfLKlOpQ>z!q{`o<$z!|ADbQeI-^LJUH>7RH!TW zDS-A98PN09GD12z)oogGFs(oItcO=u|ng0a_}w8){yd(v)1~z;s8Iz=e3zgO*{5CpB7e zJeK>W=agY`i}I?VmEa$?70;QI^EIzoejTVZ3;lFcAl*u?`#6(0MzZV=``zZ{d9yph zFq}AT>(SXtaZ!y*&D&!uLNq|0v*f*+xxw<$5W#uXE+~3)Bm7Z92Ld%3f&Q`<-$QV8 zYnJA_yC|VB0 z-^5ftl9s$8l^j!XRqvzVGw$p9?R}8j4#OaEQ|D#JC${jKsrMWwFat^%*TXgy1wATU zy04!)V}g*%?Rz0Kbxph0R&!c_z8}xL?w<^Lw$s>qbyJcIW$KS;`$&l0fM*%SUqEK} z4+sU3R~~8(X#t`Y!_mep#pjw$vD|zte8%sTyWtVNi*laAq|9gXOPed^!05gNPHLvQ z@z4}6BR6=czB465{St9n_Tl!7=6lGu?|zl<+on#FKbM?^y3L=AhJBs_r*9rCdj+zO zc`#n_mwh~$66_`P{U}oRq;zxHu;jm`PynW4{bkm0nymNPh6!tY=TOD_c0DfSP~`a@ z$L?-sCMgU|j@UXPEhtg77y)j8Q44DHEJaluPa#eSqLk@nIYQD!bFU**U{6y~X-FVzi2c z)QVWXm=fiuv69sYcmjY~wKU+sZSGqspU$`LDAzw6<1%HLh+S;F@xKE_+9Xhf$rX{0 znrBv75Nw>Cwv&lmtZTce31cS4b+3Uuf^Zb@+GLWnCG27O96FX4N?wG>L7TH$>%G$> zGOWV$rX736qv$kYuu0LAv|_!{sfZT|qnW7Y3DbC~c_KtwUMS?R z%~7@gcO&-T{<80OLpI!XQpyXa$uW||D-E$;%Yn}**3p(gOU;A+5 zP+c~HEN6kvo+zK}Su#`}-Q`|=!xE~#ryIZ#%4(U1}M9;Y4E;4yGOXM$=GFz4MY z;vR-tHmUlOvA29Fk*`Wwd7fR_AbIsPr6(tG194b-m2u3~Qjp}6!A838K(oH?Y2g24 z!YdB@`%N*9ZeB3}B2&wS_E^ANT&SVB`QZ4`SG>IU(UY>{a7O`H-KdzLaf#24t zk^B(9`ha%{A?_CK;5nz`37XK-jza_e7|`O3BY+gNDeLPE*DjM}qSd~cn-ZSql=m+J z4ZRyrx4OCyQC+>76ALvXK}Y$c1wQt#A5eP#d6CfMdmnh&^TU4CI%zn-NPtel)tXYM zxHn+q=^HwO2~d3`r3*jeCz_NV+uFLE zHxX`~3q-vf(tXp$vyU*awj~3(1jk^k?1{}fu(T(pj=h!!tBlOT?lkYz!K@^5;E3-f zDM!OHiH8nUsu_U^IsL6MFDtGu#4aN1B=ss+C>A2)9eN=jM*sh}X;)i^y=TOTJ2OIo zGvXQ6Q7`tmcU`CKpK!gOO)t`d;WAs>c12X%MjmrxveE)wr&J^Tts}Xqd86kHCK48m zy*I==z2)seEtOlKOH}N3ZXWTDM&*~g`>gz%$PfLq_n=BnV9P%+ko5h+^}Fxg^}KYq zxRpq^x@%Y`WEftjm7F`eIn*(^?+hPg&T3q1sebQQk22}O#WQrrJBpL_%oGkX*+ncr z(d9*&c2mZX12rBBgDw-wC z9K%kO1P*TsLHGms`_34R#q-lgH`{x48@(K^HTt(x`lf@{>U`)u#M*WG@ESJe*=E4z z9}rrhB^xGRuI5DIv3aiO+(_XjW*BZt&t2_{Q z)Yy0Ri*HWwKmW#N*K`j!moNd3!F67PS)R({v=h?3L0Bj$g=I}ge0F0zMb}9in-cQA zS#BdP*d}1R;v)8nmnMrwyPQhj~8)~mXzccEGXT!t7gJ<>@F zi?xLc!q%4xnWB=+?&u1}!0&z-r?G4{l{oB-i7op0ADDN?hwmH5=dwuF>7X;$+kYnp z{`=z$`H81E^9)ENHno+QGD{dY{y?Fs0zG+Dy=}s~(nYURnn3Eib`eyGZp$nx6k9Q* z|3;8vV>+|iHLdaF`HVw|_z-ZEulR0yXEa&~`h6FqbN*g0rgc&Vp~Ig$({9{mI&*e{ zfp~7R7Yp8i4Mm%&W3VtZH2s};KiP$pvw65oPm8I9fNZAi;#J5Ri#7105b?U;yG%Ux zSv-8b5FVd;_7OE5hG9T%fZ%ZW17OTtS%C5^V6UaFZ@oL*aja3 zvRt~{xlA6?+H;hs5Z{g>UyDz>ZwLl7Oo|%t^#QQeYpeBpUEr?ZnQ!}5fIS^N{OW8XWyZJ(F9 z)|gSGo_A`=^XQ5u(TsPG>}L6L61xLa+Y|z~TJ9~i?Mwwr4xau%ciVwk?Y%d?ZChR~ zfAhRzz2c+gMB`(qIZ$_4==)g2VdqsS^fK?YJ_PKE?KqLH(9Sdl(eUeq6=_I6d3NCM z)^Fr|5OHdf8)oG^B(RHrw1*N+&s1P&(jp=JUq=LqB_k9`(akel2y#bDQP07V^l2BT z4k(8VA{b-6Y=WxW45U%?wNt|XfvuvcUH)i(tFf6vGiQMXr?gUAe;E%PmS#Je{3gTN zaoi4xo-!FSj^kMz8bO$ZG6jZ*Pd-{U+2$`pbmc-bp{A=&yc^|^gu4d0gTY&6aq`4MC;j5;#i4-7Wh}O+no21< z)Z_vS*)iRmPM(6RqheZK?lInRaA~y#YYM9KkBBNe=Oi0_t{g9xb<&4K9LM3+poR_+Z0%bMKkp!?d&_yPZ;C)|?RWbDaE-+L zdQ$#pnob^UFPbQlf`+zH(@ffpOn6uJ!qCDEg<5{c(=Qk_6 zNzJFbH{RR82Z=&c_9*dj{1*awFed=sjROL6AiN~mjIGqLxF{qzF=>h>^C#_Vd<~DK z+=8ifvyOsRZ}Ok!4b6A&Hx!aFR9tDU=;DJ~En-QE@7Bv+j;X%qJ%Q9%0I&&u-(K71 zb~t3>O|Co!-Rc89M-BP4_sjY9hcsc^*BS=Vzbrr41lc_UIT9Wn)hDb^DjThSn2!a{ zfnnpjlC#TJ({0_qIFC)#TCleBwWt?QAW?qiRD_JwPB7Q}*BL%buop%QyV`@De!ERx zCa)Ks&AiZyv($v~FI;Mup%ya;V^AGIFZ*Hr5Km0!=7qh~s-&yuRSXhB&EUUQ(?kZ^ zO}&OyXF2;92>_I}4~VMs4sx~ZqgP@1ejX0u#zHHG4n~ToiWupGQ8fE7?dzhtKJKuo z;EzM@&G8em79F5N_Yhgy;xU$bHNFhXejXPX!d2CI(u+yLgS5!{945F)FxTlH8C={N`~fHTuEamhUJSzrb=PF)R`W# zx?}@~?Lf+<_YY^ne_$Yxl5fzo@^^&UX|5CJ#Fm_Ik$DqP2b(KFbjNtPV1Ct{Q1ltY z3B#kKu?0#9jLw>cx&$;gW;pfO_^aCu?ig0$aHJ>n;CDPydwBw5dY@HUHQ!y|H(@)F zZ4yvWWoJ|`>?uhRWXy}G7vXTrGUSG(a>vV60nndwfx8n!la1eXxGxjEPY8a$-V#1o zty~!*ytEd4axAAa--plo7zzA&YC96 zY**|ou|i7;a(l4y^oJ63?#jxhdL)+jug^b)3@VFCVBqm5rQ-Q%FuLOpQfc_E%5`VF zdEB-Rd9jrA6Q_;R!cGt@-!WRy-6A<@sa2yw_nWX?s3058KO@_z`Movn21Awcq7oXeM9~gfrGn z61uIXi;zJpfZ`a2fCHav%)29$Iz5e_PY^se68_^D!|J3uw8|pkrlGtWA<-kO&2W%R zrmkL{;Cgys@%%zn=n}d}Vw3XRQhD=9)+MF-2Jg%A0r+oxQMYP74GjB%lbl^TQ8ccw z-(I1|4{99)?qJ-dmlqi0lPS>nYGt>%Y7L@|)xackTRjKDtZ21+>)^Os!Ob=V9n2EB zt~S$lZ>d`^vzI$zF7P3K6!X=3vdgU!j8>(UZTblh#w-6~cg)tp5z&sL1pB5OOaHd< z`%p38Y*GIpFH(46O^g-wRrfKV-O?x%=igS|7e_O9{|V^wVx)eYXeu`htulwQO-{iio(iQb@#JZ<4#!k4tNZk2-IELB$(o@`TfGG=~Q^|}quQXSPqUZ7996+0y zwJPnHX@&@c_FYI3s{K~{Dy`IZyW!sX@+gC==O)Vz2%Mas*Wg_gd_~|)BVZTxRGmrM zrq`6@UxJx&6uHS24OfHBIIQc*exJHV+{ldrkzkzCc*97Z)_~eMETy+m9c|#;$BPL& z8i<+^Ik7O*8R-wLVLm+kL9)Q+v6nS9Y1Wy(AZIg);W*7s>aSz#FIBeHzsjx%nXr<7p{t! zBzb?)AFFzfNv9!NQ+U`rcL1q>0`m5F@s8ie1+$(;_xod2L0JM3CY-8Wip#I|zZzT@ z$17uIe36#M$&q}+4tw2;vrFc(#teH*oLY?`gLz?zL{^ zdOf)OojI2L_Rj&#-Zuxk(Zfk3ZfO^zG+p~P!y{R%dKaraj;J$8-wkG?{1CKsuxFPv zbd7wiDJ)i$H%ETIk25o@Sn5x_!?4U>o?VSH{uCHfwHhRwb6Jt5`^G8>w@KhdN+f6D zs5F6yAOEv&DKi3+@$0}Q!E%1O)of_Sk#>Vn zCxxT_l5;9^4vK+@i7#<~BRF3GAWs;q89o%RZ1_w*Rf{U5u9T^7_;j%1-!W~#V|pH2 zrskP;-LbFb*RvN&PBbDoya~qa;1$xWOhx#BAD$U!;&UbV#7F-3b^KfYC_#Q)r@cpR z$SN^1@4}R)Moz}eFQAV+TW^_fFQ`X(>x~@@!Ukc}%YFZIZc%3>O0S)LRsS@wFDJp8s82Nn+Cp69M(k zjSQ&6@mTmG)tF{R;qeT9?9Y_r1=oB*5}Dt3T)%(M7kLLM*F8dZEOa#MD0(m!YX7`-7z=e(Nw3qpmpfhG)^4N|paGkGc($xeF<*~Y+Xd5%ih6^`~%p+@ev-n#q58f?yIqbW1 zvF`{qTPhao=-&O^SPM7^*VE>nxDxK@baq1a2!icapaamxu{>mLlOYkUR@a55v5zu zVWhGdiXx}M(jZSYIRiyq4#!KL>sRR%4F;-A?v}uCdW;VS4y(H{6?NJq;5bKVP1wcc zP(TWdMXHhB|DnKGHzB~cU^I9Kg|Y4JXSK8gc8<9(^{-_XvtDX z@haY{1X;#LxA`9T$sl`Exc}4}oUYTKJbygmd@I*-k0RlYkMOzznrnRPo$`wLtbYXj zu@-=H_aq^R7-gb$nDHQEJ5O4bOLey|QmdF2bh~64<3MB$6!=&iA5!5uiO%nLY{N`f zp=4*&0_9G=_fbS-`p675ZRghiG^YV>br^NIhyat_ch17E#P$~hx9u^MG*`ke1m)4 z5g*<~gPF1evG~Rb$CrWy1`YCU2$Z?XT=^zI#%b?)T`_ry3*&pf(rchK9|USlBFE%14g8dGTCs9Tf)w`;;5X zcX$8sx5Y!zd~VXhLmWU;&83XtY>GmT8@?0i%xYRQi{o}}!uPxh>i8yQtIR_}`-$Ni zClZ)3K(h+LIN4&+1a%miIPT8D(MgK!E?0CWsp7ds0UoW`JD@oq^Dd5vWNJYo>&ScK zI8K!GU5#S{5eEq~Sf2VDzN*x1B661Cq?-9rRjQkrXLvLz-Xx?{>n1kP0iH!Hdt!71eF^mh-JQ8{A zbs3A(Jj^)^$=H~n;#nWqgTcFhz7vLb5@$tfM-#4egj~7OC_ZcBtEHJ&Z|L^^9F zo)AjEddG*#Ga{N#D9MI1qwl0(VW-Db7Z~r**817Cl-2+t*zbLoBHZwwryvmh;XJ>m zrJnMfLceKU4MoFErX z7Ymt(f#s4&<|3%}dqz`NU*1Iv9Z#WnDGc~K(<)J=m+c2`3?i?yOYIMDk4XiJ`+<^# zOK)4a4Waz$&2bnC{`rJO(4$mQIEDdKsq7gL{6#GzcU{?B!iAvQKf$M45&yCK$HP?IQiDkW<)pz7_Dfm|RzRmF-lBUzM zy0@s#N&Hk`Q=m}mbXMgV&6y~klbjrwE;}RU5T5975y7+$K0L~=oFPa8(EO((TK%6L z(Rwz4%Rt1=;F5IbaemoRuVPHPa_|u^qxoaC9uGk}oW^>pzNL3DgEvuB6 zNw3Qp5huOigD{F%gQ*Mbg>U`e%NS1XqY&^0@yZ~huW)CxIJmy zBh(Cl00nr6vh$?epJHO9T}A`md%6_C418rN<93`5Yu@C+?0#)s`${FPkvh-M)t4Ou zk0_>q);|WOAm?)+{2O!IbdCHQ!+FJ^u3A5m5aFO3mt}5UlaH=fE=Tx{Sn@-k0ze>T zwtUIU-BW+l%Dsu-zyqTRI}V1`I@ZJl9Kn8q(wl;8KLmx4&<&euSVp-?P&VZE)1m2Z z%}kDMzMBF(hP*?f3F?gHKQKJKx6)YGUD<;yy2$3N(9&Sow))*lxEzyb0GSzrfpAYi z!X|%bnH{KqS}JItvHkD^Rq4Zd-8OpuCZH-E-@<88%1d*D-zDA>N{@;xa#Dsl z`$|6A$IsFU&@}D0j;rszY?^MWZ-+B`EHDGZWnmL(8}2-k>spel!SXlPAWt|KDQk+mLvVh=+n9#5S9&=vz=&K7wBF2oKS-C z9g;#OrzLQ&#ULdu0>d*Ic>%<#t6O8tvk6FlghaN9JaEVcwn;A4rH-X!?SyJT5Amfl z;$Xi!Wb!VU+mTZpZ) z=Kew%OI}?tdX%l+D1&r}vzCIdii zQ$sv*tPC$Uk?|mw1_nOXUr8%8vKLkUr||#h*A}1c;|$Pn#BQ`Ee*3xcLhh|>(m9kG zZ6=HOb_hY;G-Af;Uj)%7mjp)6wVc`E27PMHh}LTRAbQ8!{<8r-YW@D1+C~-7g?4>Q zqHNLV!O3Dm^*l143Ze4@9$HrmKIyw%8G)`P6D593oDu|ut!EdZSnJ;4umxjR+ol%~ zxQozZ_wGN}TX(tVSU)LK!fi3@nLQecJ;8=;4Tc5~H&3Pd8LZ*$_k(uin>A3i-DH=K z@U%9ru}3d9vz2lu1?Z``Z>e{bvGmLArN1SP1n9G_%MRUV8sI7J#*j)lvmZUQp31~J z=9cS>&jo3Go?UNF6R?{;`mw8xJ+laQ_N8oK-JLw9SK`Gl49=UGkn zsMZZ@z|=(IWJ$+eR9cEH>39={c$nQ+3RQO5J!f=8W<((~#F{Iwp0L+HMBJ%>J~rfP zJmSsg2MVKO=TnV8m!F~eRzqU$NptpnQaQB6%Nmyq3On6E7N)(S;6y32aEe`B(iUuR z3>~TQ?v{maf^U{|oMED=$-qUOm|*X_Cj@6qc8*+Q?hP@$DBVXfj=lh1g8xd0?O&>x zkfOUkiiLJD?5&pn4{vW36=&1EeL`@8yKCcajXRCIySoJ%cXxM(1b24}PUFFy;0{58 zhddw8JKuck|IVy6hjZ8mb>CgRx@zy*b?x7knQ9hLpR-j~Q(|IcB`lC&mLH#JHvC+| z(t%Tn_o6^F2 zvTpxfRbKmxcvy70HNQA|A>ql@aNUt00Ssh}hoQ2r3(c~5;1ParmQ0KgH~ve)&o3`{Icw*YQfty9L5Rz$q727!_x-Pq3y5Ftt*u5ZU)~E zsdZ|YzRv`0bGp|I)|0gAd2(C$)-RO)12rpg_4CU8Drq-S?r*|O_4BU8_oT6`05d_Te8m%Kwl%(+qL9V#_m6kq&6Jn6{<YBABG=E;F8(3;GNUH z)Nsadv9jDthr?ru58I$#JpxSv`$$|kWiJql(Uuf{%~liVos;PdvB%rwb7Y$4SSl{! zaUet@$M3IqB7L*=#0HNJ&6UCvBXxeTC*kXjL;z>R86{??BK`|rLancPI|Z#9m(Xn+ zec_hte-VEgmux&y$L~L)mv$jjvqT@pkEUGmd9tpTRWA@zpr_=q<9NG8owWiHv+p5M zz2Wxv9h*ly{#k;zn<6bh7{v3jlGyMu%hY%QCefL@_-VwKQQ7b`C9t+2+^5fYj1eWH zYGIdPbqUMY8X@n$4WUV<*-HfJosH^ntVuoITAB+LMt(+Cn(8BGzMZ*Jx!T%qRZgsG z3Y!Zfi|uW{e+DdJ#l#3M!4;Q0Z(Qrtz^;PyuW|w zh^Cq_^aH2UX4Khi9ENaf@Zf2)oqFQl-@!WhgC-Ks{q~~{j7J|qlyi`i24HP}RHqM5 zDdDc8A}9f2x<`<-Tgqij(hD`-9uf6ghUfnVE$~{aCC>3AaampIP4ImtFhv?>?%>Wo zt~}Ef>)0RIs4=ie`J!*x5_Y#RTl6(~PYLs}b-r!*PB|OZ{%Bu6{dT~0lJN!Qh0u8= z>K9jn_>#8nD-;|45S7gd7k{{>wj^D$;$iffp6$qb66ZG1)s;gn@scDJHNfsn0!>cD z|M9s0xXBFOb&d^fA0#V|BgiwjkQ}BM7ii9Tf&>}`rXX9tAizTN(bLvVCW!S&t&r8+ z)?-l2@Z!46SZmUO%*JD4uU1wT0Q(6W!zAl4Q}c zWKokTR{}f9jLG$wpc(6^r~CN0%HLGtpj8a?oBN;bTCTGKm-zN1c6b?=iECUj)PezF z94Rr4&vB_bF}5yo9{Nor%X_fN;AWIM-xmaQ+fa8S&&t|uxID(XL@cz?gB+p0yOV3Z z;c za^lT~tT(JYv*mNye;V&A8`98yHbKjm<0dX56;Y94c}IvTBn>o^*zD8gc9g_qF1>QK z%u=$cc!}aRQiPSJF(*u2ZkqF^qoF8QZLdkyZNEDG3V#3LS3rIh={QF~_y_9HWN}`l z=L{9*ZGSO;qF>n1F|)%IPki#ohsgWU01wtS0YpQsLueUYFBL#c@_KyHE_-`BUm;QQ z_`NTAIbOgz<`ebCPEBd^(hEemH=w?t=?~_F=zLeLm%8B4m!{c1k40C&%@_ufvL-5< z1w(vHU=_yI;qw8NLX&%VvQ<7+TISv%JY^Dk9MO5bVYI1IX9qf@cHwCj;1wb}hs9dY ztDtDLsn95>60Nk`m_tQQII4Q8ENTB6PmW$%l(Jn}9ptVztYSSzpNocRx|H|Cl1U|9 zy~bZser3!jaqaKM8c=xm>k9Ypz)=~yM!Zb4E3sUT8HSLz4$(L#lJ_%({SQvEOGU+Y z@!Lh%CG)D(`pWe@CPWOEEdxAiOxqcZC5xOpADsun{!N~q=YOE=+bUobJP}AIZdw&^ z`v_ykFftUs-RV;5ndvZ19@ORP_E41POe9yTU*Y=C$e{R$TVro%Q7)TPAL)2B4Z zO(3(;1_DF>3?B*~cxSo~nKJa!l{7JIl#oN*fR)b}C`Sf<&Tv9K7b7_ql?V^28IK)y zjm5&bur~-q+Dp|lcDC-PeYGTKS`M`n|K>*z3SR2I*^sVI@m-E}mNUN(lL4x2oOLvL z9U24Dq)}43-JHmg7>XH{Dp-p?f?wTWt!4&n(doi<&n{j z8+myZL_gb@K#pOZv(JT0DWp=V)UK9df_#&)f@E6>R~waQ&B~72b-&bzwc0+XDJf*A zPi6FSBD_G1dfxxfAJbn2E3y4?QOZSR@sOo<9G(3PH6?rh_ipEco(u}7Z4-g3P$6I> z(uzo}je(GyiUz1eU&1>lya|AC9VCJ{Vdc4Huex5Ir&Bmo$SfwV#k3-h)Wal>xNt$` z`zx_n%ABaTHEXojhlctqPnpd%eujpjk0s!A^~x#*2dlw15&jII+}+z%wTx&s={Q~E z7=K8Hh}@wBay@nY{IbSvhCrf3El52nPqW&o17i7Y zZ}Y~UDE4^8zp9XYV+$&j>zOHi51(t@RLeP-_8=Z`eMaWeGaLd?Q)PYd4vXS*71)YE_wa0W;w~Uysw>Z&J7wGbzNS>p^Jvs6Xb|jDJcf>>pb-_fqr3AtxhH1ELrr zEub{gL#^Kt;4D`2Ft2E&^vz-&lBzgr051vn51G}v-JV7R%MFP>$_SIPHYB8xZU(B8 z;|A6!8i{vl-mkwLX*&U&uwp_@7*kkEEdU533+H2#+Ex69?)7iRzPjW#c^aFV|DeRD z>)(b>|AA@=3&_9H))^(LNpsubn)6`=<$k@7XCP+bgAHcr(n(ZLgk;!G;6qtcmiZHi+Z_ zY=+^PRx~c7Gt)4!=5bN-RHHfAu**u6ViD=wFTp1U%$q6}PVMgwb}C_~{y4wg-v9Mb z_?ex0Q{sR>-y!CWh79k4d64djNw0^HBd3*=D-G2Mk;n2e~OE{+k z+~uZKV2a*PC49upFf&~ne3utOHc7nQoA&nvaC{z1e+bnWOl4Q6z2NYl+CWF8PthcU z?`1di`e=FLJL2ul^1Io%ZoaQaTP~16MNW`E@=kXS$~&_A3dBw_PJ&qTQx*9hK2yfs2T)g^G?aY!wUvKIFU~mTekmE>>@ej2&|?k?)6M9q^)kOTUfxMlMiS>9$o0UU znGYW}o>5yXo&Vw2R^W9{RnbRh<^)O+vO3<3fMkh2CN`~Q9I$p2m(?`P`kTt`bVjxYZJWIVsS5)S6+0HhtP3~E| z%*O)kx|kixabL617Z$$3&Amr&w{Dy@Xc&c$5n2$5Thx75yM?v7qc_`fBDW%E76D_{ zNNq*5maP^`i-%GRT95Pb>AJWhl8@7f zg_<|fMK{asl|(q%GqY!|*4dB9r+~MEk3E@hH_muLAeP>7_X63gWAjr#X&x?=@qr`% zqvGAR^<(i!Npeijf1oHTV%u&Wi8H^&%=1IioTOM`{A%jMT zePyqyYUCl-v-BCvsy064J2I-3znxIKScy+|A3M=5`F~fS_|SADCp(cQ5aj)+qA^C5 zYJnDE=$KMsC*c{UD71#L7@St;j`PNk#o_EAn^H)F)Ow?JYQ0fyD#OnJW|O&U0Yv6} z2HV4A*~Q!Y;X*_p#xU(+covR09MN|YS-;j5`-ix($;$019VGhL(<)#45C3HC3={-@ z-v&e|I)*&nF%{KOJEHG+`|Lnjd;TNnEmwZWW~*=gIsni^|Ac?8`5A=44h;7RKm~YvLrDQm(roW0W2Y0CEAKTTWoU!1pAwP0l zXUDg4NOiF5=ZV86$UY!?XFIn-B^=Xy6@ziK`^EF#zv~bgGyPktqep5nX4RihwuFDc zCdk?0q<}cN0UU|P($Jz$Mmlj!@gj)u-j;Ixlij$cYnr!8$}R(FddB7N0Bowl93_3a0}6w z`zHFtQ6C~+R@a#*^ex$$q&O86LF*6w!z;ziEiyA3S=VXG_w*#Xq+y&tO~lUltCJkT zZ9w+&B}^lnIrC5SIIL;!Ow^R0C zTLR5C`=k5J%N$;#gYmfH&zDUS%jc~Tz8%krF>qSqb%2!Q zo~o?ZZ{5vG++bB*TGp|UadPy=bw#c7S}}OwW<2pdHTk7dB;SJUKol z_6e4M&y6NpVBALCalmx&wA1(fB7(Jzo2X4(Yo))niTk?y!}jw1*Xrxm8&OBl6HUl> zm93_KpiBaS9V)E#SIO54oVcaZ*}L#>^%Tt&CmPQR1qx!V`Ohu`1TmKPXca9b)^f1^ zp0iUaii?eoLr#liapx1w$IR_{!dx|<4XKU-?L>Z ztj@?_&IcQM0kdg+Ad(B%Yc4K4>{|PQQ147l>^?pTL!jZFR2>>whLYnM#qbTbkJGxh zx*~homs>W5E9=Uh%U#1x=;2u9mkb_NO(2-}rpM&OjS`e4luYS;YqA7TM&b1l>k2c% zg;%pbLyl*0Jy}ZQv9l__#<)Ipwult?SbFqy3Zf@IOyqN!-MB|FWJ$20W*=uC^OC{^ zang399Hh|EQ0fWTOS#&KBOt}Yx9lm6^3j&KUSNuEuF!WiYWpm|J8!5$7w0jBxWWg- zk1P8BfJ!=M_QHV-4i!A`QdgU*o|zWc>0;F&B(7^6Sf;va*Xu}BYw@tPe+xSS8sDD6 zf_WG5LSs|l8KKHLvlrhYb4hMA-gI*&+|np_wr1|=`AQ_dLD(62W;7nN4vbp^S041c zKnAa)y%aEH#@nwN!L&mSX|cUIrggB%txt$m!9U7s}(TF3l)ojcXNjTMC_Lt%QJJnOD;612eKL%>^5YP@JIZMlA*%8Y7P z;wi-N7BZmwLL6n_I_c{7@V7%`tnTOggz1;XCpyWR>z{3O?aT%Okk`vQ145=a1qv=Y z^FPHx?1oYBj_L~v(kl2i@O)E z5pm$~15%mW{`q4wjCGg+RB$3&?0(tp;lkcS8$Qk6^-&j?-aDG9VBT|DUf|Pcm*(bM zG3!aF5r_Xt+3V|F@Wp0hUwonijsQo2OQBRg(4|nsHQ(ZfQtfOygE*~=URA=yKZd^# ziR-wq`O?9SC8CA$+`ifNL0!Jhx>G%DN)vN1;K#9f=`bz{t3qreCBYuua6zl7zaPnm z-g$M9j1L4n(re-5j?Plnt?Xu*X!IR4xO883Ppc#6wVT`Us@l}^H&e=6sl{)Um#AZ~ z6jz*zqtv5%9*trgI{-rW>l3sM)iSv>EmJx$mN;}bIcS{)sH&_zJGxb)HGaFp;9RRP z&NU%U2gS#wNFR#x)L_JO!#>MYVxd`z)4BqTl4zO3wC8k%?W>iEy2WYc)km4z&k?pf zyrC|2YOj0D?}E!pz_vSa_F_ml?$0M7vmpZHPzSf8L%A3ryhFH5Jn^1LfQ*Q9*ccmz zUQ4>V5z}Qyw|5zXyf1?ah~Y*({S~KT<$q^CM;?MWca9hL^(>K0wP~HlE;EEcM&X+H z;7BwWSY~Q+eE$zL^{8dI4Iwss-&WV!kn3n}m$+(xG!H|uBI+sWkp@0|Tj zoIvSFadh*m+L2~V>H9$k*cN?+!s8>T4W60_6m=7)L=a+BTCSZ$ z@V`F3B%s<`cxk|KdpJF^MzrFZr zu`!3X-scHCycW%&Jrr-mf|un?AEad_YUx)?vodC6no_AMo*6{r0Y6A2jI#HN_U z`=n_{XBHX6GPM^Ad(giK4v+b2UT?9-{fh>b)1&hwE)qlPQGR#|hD@>8b_?RS$NeOF3K+wgc!Q%2t{Y2{Ey1Woe(ltXi*+2OS`GCI@zjCb&N#KpUy(M?oT&yv{qA1L)L*DRD4 zTf#1Jg(I4fBMCDqN_aGIPxgWOM{~^ZlN1>b#{4ofdx-ajL{Gg_edY1<(!l`xNb0-O7lys3=wCT z7vyX6kR``{7+M%rK^7M?LENW!JeSc;kENnOxAt;JX{3!S_mTAvnB?K7!)#H?(d2+n zzLXJC?dAgpQBdU4TX0=?=g>@8?DX-5YjAmAJGnlQBN^dj+5}|&R)L#Z&Y{4LS46Ray&Vb29mqU-fxv?`goSJ$8RvcPH-ljK@}T z_Z{gHcd;@o4)ZzPrC3f%F!#c*@m?VF7d;j_BK3RPN)1|q*V2;)ctst zHDFuJ)>iOUN!7F$4rLSv(3B~D5VOg(t~`8kYRqESB6>p0`bKwC21|2LP*12o-r0QF zt|B7}>KUsxjIFA1iYu{N zr7es7fatzC*C|jN-8~Ptqy&2|mP_p?=H}rXwU*~vm3wM)?;*@AOTDvwP9e)_|F+b; zAOca~wY+-xq5wbD{aD)2NjvSCQ!Y?ceBk#f_s~-5Vx&-*EGWiMJXcrsuPGM`&hLEM z&xtU6x?ncI0$vHCaUR%t1R07?ElnLP4i(BS7cDy{4t|w z(4n=tI%hPb^mRE8SgBilZNzURHSfHQ^GnO>OB45H_gh_i8|dlc-8mq^;QL$Zz|V?n z6)#=Vdi(E=+0LNMhq}v3$!9=pN`6-C9s6F$owi`$yShAR3!)3dLG|fSXj-;InwmCP z76rvK&)nQlz=86Uzu3SiV(`;Gc|VR;TvG)62W~of0X5V5kKdz$@%jON9f@vUWc>!8 zoD@mB5%-F?hiy_r&uzld?N}FDwvw&=V#2GYFovT z1uSr}v6fv=<^k0iPh&XIK3XP>Hpf4+)&&y(>hAr$g<7ibT0^TW+g79Xa1TwMhVD!& z<$!KZKr6uLVgULDUUJpB6*DgowcMV}s*kf_tc-Oq0cWl&T-VtX8vCdO3y}BrjZ+yi zm{}Z|do=zry2YF$W7;zD8dC$u|3v zQjH%GO^g0$2oN@$r-@dLZaocq@J_OGkqf!Sutn9`>Kk$`OBYR4lhz&cFAy&y_ZQ!l zpI~k6LP5;)n%Yj3a(=$BhnMmu!gQgvE@pKtl9fNQMB*gL;l42#C!g@lu*GkNihs5{ zKZHFxlpd9oY8b&Wt4}$9_05cnW+7iN68z+?hY=-VKh}@PjnhL%t$@hPi#4K`f~V-V zH19nShY(#me?ezkdQ_91A?|*uC3sb#ar<91#Bfmnl=NW$?XSwU8}o|j3h42298&_f z<|GYFl3BifNlFGkZTkAYlHg!G#l5UmI_aU|#6YFHN>1B7_v8CQhNkrk)bqb66J1ZH zN9KEcHUy*F5V{3w)U3dn#<*oEdqr%#`JRr*7*eYDWQ>9AR?k0Yp^lP`wnqm`s;cbZ zBZv^)0IAzL{M00I1aq&YNXld|ehL-;{?`c7R2zR}X#t!RKg>=)tnu+t8uZXa_dU7@ z!m+t7^Vm}aYQMm54SxOXWFLL@JJRys(dXZ-i6n2{e@4;129nDxKL;jNoUUfFoDR1b zS3U_zn~nr{nW_Rr@#~U(Q#kF$)nRUUUuB@b6gV%BK4QUdxWP$ zZ|(0DF}I1PyqKTXk#MYphGez%7?BwWaBf23_sqzDG=m9Yx*^1{Bx1INf+<={75|xY z(8rXEP({dB%t}QOemn=FN!yb+7k#J6v?e%J&C?2HrbkFi^@80eqN8kXqXR5T7d5&Y28wj3=|1WU>q?i3b@ zegyO@s%b2$iw2lq{-jX(@)~RD1!83y3Bu6zVtSA-BzFVOv!AK6qOp;e z+92~`p(WY)STE0q`Np?-oMHK$AZ`<2pAq~EasZtt_R8_KsZVraZk^Y<>0gdaX%B93!{VICsx7&j1Rt7ZLcXPg*Y$G{HmIxFSmDGU~ptz z-z&jl?a8LaHd`@d%coy+l*3>5ghUCKTmj2ho#@phv)D!+y6J(W#nQ}$c`c`@p*tia z5O?>Eb!fd8-2Mu8M7Ww|DoBa5f>>%GWJdT`T*=z|CebBn0B-a+byG_s{%dH=;2rak zb#oZK@tI3yjxD<@O^zW;)90yPqQRro|68(-_8~v{pNIB;KLs(JOh+_(E9Q-rL?+tr zeRv|~Q_ZL0DPi^vcO;Q^_I(sS>_hEY_lC}OW9ipFGEt|GEeYne^91ns`!?nqPaEBB z(s0oftKW{mr0x{E3`@4TVI1rv2k}lfp=AZBLdm`D(Ma~2Nqti5mTd?}maxs$gzuuf zK=svoo2ZlWoXwX~c9Zm=W;^`9eQbZ(U1$HE-5(fUd^sAg`JQPT#AZ??#Lhj=XssBN za$7iMO+t*A3)z+pH5cdL+xj7dy$zT*OcqO(L$XvBK{9hF>OzUJ?+Ro$hzvDMp)Vnh z5qKi|{zu^fUDmUR$O#Z7WF;KV5hV}Iv53|z7DB@&!%PcJHkBslr@iJ>8l>il7PM~? zH^ot_Jdtt2c`o}UljuT{t^f4zHOR@L?4j$aH;j6h-R`_?tE0g|Dgz+C1S1$K0&iU= ze42<|$+ILPj${V6K!oCD5|s!j2ijuJ)i(0rk9+ugnzUY5B^n<{@>kDAU^6jL8C5v+ z@VT*SmM2&0js$Q;W{710pAwZh8owbF*2H#=O96EgoQh+v7Pi;GIKQ#vD4y5u;D^An zqCAI{c3qw~WJznzCEU7V8{f<)4v9fAS1IH$MM9*iU8+`rCq&lP{E^&@s?~m&-|WAR zK+hn*R;_I$zZkM8rkG+Vn`F%s8?~_ zO8V=q_BEDm%O?G&1J^%p5d+LZ1W@9IcuxgDwrx$2wUR^74ex|kgq=dG!O`a<#=MnC)y6+ z$$@+t`S#{io7$xrpT<6aI_)5A-qH*+hR9}N{YfGcpUGY78{I69QV)>Xfc0$hX)Tq( zxNK$9SnevT${%Jk5jn}6dD2+{HcbqWHR`5cplkR>>(gK1qi+He z7k=;_Ry83Wad50Ii=RuSD!Pz8X!Sc}Ii(c@*eNvJs2tVx)ItJ!h3theT*Mv$1GX+< z$=xRnr_Xu2$%+52wL;P~pqKtl)k=!vMB^GS*2D+j?G5o*af+RuaP@0}P#}L(dngUv zCD1AcC65QQE9- zzIREA8)RGgjKuM2gw>KIZbA8UzIfSjFs~xc5B_RJec-g7|JgAr?Y5jMp@wsc5Dhh+ z3bEH#*iKtAGdMzeA@J`-t@)@%syYlj?d{Q{iG}oPckwK1`-NKb>!!ZOz;~1D==a=> zwCC}?xuYzYy3UhZbO2xDu{7LwOIbNur5H4N;RSRA43gW=q+Ed$7gjiqXKCe=w0kY3 zxN2}e)JId*e_yws-*sHP&y%+ps0R`J;s4XT>(RlUxw-UL(~hb@Q#cl}tP9nvdF-Ric(SGOFn`1bJ$E zbmE;d4&K$Vs4BVkWh=Ye9f$>yyB8&Kx2GGdY(=)sN=al{AY{V4HVv|vd^fgV8cxaw z6mEH(ERR*bu#j;!m=}D;QTTtbCDV&8ipJ)u{+P5scQB9i;3Ql-wqRw*(16%$!5&f0 zT6RL>_gGEqA;9MiXM5EfL!n8cAKw_v3T3y5?%Jlz<;2h zE{vSPko@iLXP3!c;LoMnUlrd47^^b@nP=zbu~h+8<+*;uQ4MMezv@_;rH?%vnTgw>qSBvs5i6w6uEQxxAmg6r|EX`9lT3OMgpf?hfhisr(-29C#8Rs0E zau6tu__dEf*iB}KIrV47?ZwrVy^aLVch6*4|3E#HEsHh^oBji}y9+%x zQk&|GE5>hF=+mj*w^tK86L}b8lBqv`%#Gu&W=)EGm~srK9@RanjILIccWu(?M%k$; znSy+{qE2^_Q2v2}V^zT|Rf9iFi!&_U_h&UYXSwup2*D$_vBL@HH&2}<_xo(ZcR5?y z>+Ab{rJys>QJP`rhPWryW%sD8F`L=bA$FS@E_r@YZMvDmlVHf)z2|t}{_&AnmCizS ziBUW^@t22zqxmi?>i>Xtk7Yr;4oWX!eho0)xgox#|1Hw~E|bRsZV1SJZnY0G-xLIF zq;nZ7te&li+0Gc|b5g-_Lzrh(2fJaCGxhV+(>w;hYf1u{$(qdq4z{#%mDNCX59(9H z^!>&?(;qd5-`}5_*5`KXbKJgj*hGjFN`jd=b&`P{+*Z1V-p_XnLe6*iII*0$sFrQ` z+Cr}q6fL2}YsuYHn7+HdfgRdyS^!*e5Q`rF^i@#VyY*ERg87z8YMh*_V-d`EmwN0; z_6Bi@8`bjp1hiZbUQ)K7Z43sIi*3A9Pmi3Do%rFwKrB0LLBd2MJHg1kh6?JJSY?B| zvFV_+KOD2XPs#e5kDe#!gPNb8pE8Rs;u`X)SCnfdJ#64KC!oy8gwhquv3PlFgQzUS zYRB5rm63Uxv=E|t>q2F?O%XvcXlN9+$J|{Ptp(|Uq?;cQ`m~-aA@3s;Ds3TVo$};EE3fpD(>h{tMk$39U)no-q=fNVVT;4sXb>)P zYdNkJzUjWt@@8W(9=G7O-ULESndMgdg>-^&GJUjsKV?l@D{9qUj$k7DI}fZ zE8`U#JUrEqj>^XfN!HuLZ)~1V`zWOAeOB&j9oWCRb_At%e0OX!?(mGLHJ#cKcd%Js z&liPLWjmOK{r=qIFHL0h57hh{)61F-nl5ktOoc#3l)H(3lXCU08e7=@r8Me~uX4Cl zyH#MeX#yJBbjY-*n}>Z@jAHG47o~}gQVVQRhD-E{qAu-vu`t(%c!UIjv-6(lc{|cS zjs5#g5&gYppw4PX5*e!bRRgVfi(?Co3cIyw>^OD=NsdHhVmc*HB)Z-K7_UD$7#U>a@0*Z42lf`!`%mmJ{>n($Donzb3a7^^;9clwy-*7DBv-S&JIo=%V0t zpF;@|wr{zeIFMz6L*$fm|K&b8zCvL9sM8~{jM26oUFYVPUc6FT=2(`|SbyrlU%Cmi zQsaaO@i4%2C`5^h;f*_KpR^yHg{O zM*zFxe=G>*C?^dNLfF9}YBN6HG4>ux+cC8HGSw$krvtzJ9@&~*h=ps`wFQkj61!qP zCif-aw-wCg%#Ui+q%#EVQunl#bKdWnrwDhgFh~{4Y6a6@ItvzYsI{ETWVokMnhiw} zGl8Y5)KzpqCD4}!tXBtRsm|NDxaw1kq=Nt1a%T0}db?G~nN^XlhL*oZNnrX?{7`>` zi;2v{&~?brOCeBoTL-fdV4B|I=Jr`$;pDf0Q!80h?JGRe(aH`AHR4+O(Pt3}-6Gsr z8K$mEVw)u-4)85yBgMq94llu94;5>$+4VT=laFM%3`FQ}=U{C8 z1`b~J=c zha_8(q`(x;+#B0Y4AzA+rO{rl%Q(lIzxI9dImE#+{5(3My{O)KGkl_g(YHsZg#qjR z2I4W7E)Vs3J_$Y)u3zp20rZJ89yOp2ko-Io$Qy7VK1rlQrO_NgR$Ar{rs{G83o)&} z1&k#!ylrjxcYCRS}__JqSPwf?r zGPVhguER~F<9ti3M$0=($}|StJAWgK`;)an}x5s*?s?RPs0N zhA2&LhBpX{R!j z<{Jm&zgUJ5ZGXJW=}pk>KkxDzI1)wp{^+w#{)kX#a*%&8Nq%o#_{~>$Rr#waKLCv} zeTp+ta_TU$A%(Mc25(=E5RRlX!<@rYt%)@g8DJQ+o%VU7n@2YUp^S{r{XF`$MSFQ< z{U^S0VV=#_w=2@GqdZy&V1-G_1}v12FdgbxHTH=d%Mu`iP|@e?A4ZwNZ8tcFEQ}xw z73t!dA?*q?d^2_9#6aHu?tU%ZmOq|Fz|tdu*Mi&K^AjWBD@~*Ka6qv0we&EGR{jvj zG}tMYgbp>0lZqp|HrpzZJ-Yz-p6{jE zomZ52Sym<9Dy!x7Yy*oD-B6-De~(f5FKO#_=noU5OUEX|P$fXzjv0~tp$Azr%r!Z_ z9+o9U$1-xCo{HpJB15$jcX2-=VL?j5bSvFXA3(rG$VG=9#A3*ZpU4p0vGpSkcg6mUMos^EY{gSzLM zz%OjytK6$p?G1PqlD^e&k_F>>y`5VB+OMz4(%uI+3Ph5ctbd<}^&}@0qcys)U3B;= zmz4MkwxO?)$~RVHTR^m{b(*#9lM>{8uD?t!3IBmYDiHeXV%Uiu7o_MHvr^o{kn==9 z5=5ZTB9>h)ny@#9<#t*l@;qKY{xm!gEHAi`L=xhx(V z^5uhh^KZEam+_uEZ@O4m%|cvFPcst8MS0!N7_K4{#{NfxKOeZo+qfub-W++EPm!|r zK=b1?KW_`H-DRfH^wo-Srva`-kEDaYF)F13nK<{7C4- zXA6V)SYe-@(=4ZBr|SE6;J6B=HsZ%PzTI8A2qR%y=7^Qix*S{k8q1(NSaeg80M!nS zfu|uHc--Ds5X{f4&9P{9U`-ls+mYlhIoR-TFi}e1Jv9eY=SVQRfq#b}TrX)& zjn{9{7hfB8YJoDf-pSFe zw_&*o``c@FYqm?pVo*}IJ4SIcqmitCY+u}+Q1H_LmC&gzw%tlI(b7^-gd#C445IfFugir1<)o<;#*j;SPH4SM&i=G?2`Yw!?@kN%(L&I zm=yG?B$yR#T)Q`u8vKlpGey1cZ$v$+F65VB`rPLIva;?)DX}q;Evqj@EEiG~$M5Hn zCB%%g8~lWXr>uyhf~ETvD=ZX| zW~och_?!`QO_Xw`mPdLYh_$=RaG+PSr!zyTIHgLvDs+saq#0Tw=}=B&=t8{n4?ysw zvV)qAod_wr%#Et{Q(K?Twagf`b%Kx*atE34R0OkfcfaW+xSVXLGYGTVc=&vO+9>IE zGAhl=Wof1gF(xK{$B1K9Ai(7f$&j;w#mem@i>J+=Qb(DwK)Y91=HHUyK_WcU`@!_l zV&K*rQUGUKQHBsg>pGYKcPYB(9u0QHta)GUG~LvG1mD$<0yCMJAQG0cZK4r}qr_7& z8PpA=bF>>c!DhAt#vC$xF{OyxBv&Ek2#MHWa!Lr($3cerK+j=K_r>`<06^n-CDmCB zJgchv@{l%TY`cjQ(^U8q_JQO^rpBd#^?hN3*$-m!Xdpg7o-13=)UYmsBWx}2PXZOY zWj`=&4E$Fml2G-x+|}y3*$>2=cv+}Ec)d^5$^wE|@waG5w?fw{R7Zb;K2LeW3)eqG zen-^Ut=Oj>7x+P_ntodm<9cf@Cp(&CTi*M)#^v$$g@c9Y-L%UrRp43#DWk_Tn3#~U zZcm`OvpR}wwM8Y=D5Q~<@pEY`%_i^z^UeO`>pk-|H6sLH8iyI?ka5R7qD(MvJUQhG zyKmz<%6YJKN0KawPD+;7eZTBWY_-}J{7gRKrbx74hm>sAjAfgSay#I5Cx)zUM4oks z#BGf0K|O1+0;GqnQcEJ`4!S7L?+35!#Vz*yXLmyN745cB5uIo>MEW~ISi#P7Q#Z>6 zzpg;hk%<8Kg&&hf%`TF2o4wV3WMK$ahac!mI(^R<3x|4vXsq3PA;r4kvC3lAeIa@n zwxUOv(dY1zez~zpG~BFOhgTXO1kzC>st0IPw|~g?$;LQQ-3(PklE#jvQPeh{LIW0J zd4uzc;E5y#cTRiKW8!2t6P!uVfJZJq8@ zza1NdxT;0FQhGUFzH(G>D!VDgASE6VlzncTqE>A~0#`)q- zj|g8gx(ON#?HxoCPi2&q#U;kWArm>bgZ4YwU*7E7^eal-RU{-|t5xBhNWF|epSZmo z4QZML$SBW#PhS$}^VFWgWY7(BJ&L6#DJv{~mdx1jT}%D7(crWKOYjiJoi)%yFyHbc zMk^BLX(Y%^$#Tt^dx7xu^<4`Z@#A8&1G=_DFwwSV^XULGpV4-J-2REO|bUM zaSu=Shl7Jb(_)TP$OtT2+AwVhr2>9nsitDTK>-kvq*#>?^#u~POSSU z`gDNSc~z^k-LB*rt*zI@&mOurcQM(M8cF714s)NR2YNcSHr$WRv^T>o+||7kFi|T7 zQ24M9^G`WjxA?|@XnT7g{xuw^x2Fz!UIPq`oiRyQkv5oH^Bmu;t6xM+58`G;K!RVb zngwXvd>UKG-cYmKQHSi^F(KipW{*%|QNVwJ{GtYhb}){~!bX2>$WBU)7cv?V;{umC ze6*^z)v=b@>1$`yeaHm+bG0)nhUFK&qcW*PW*pBHin=klzm>nEq$^vV9AoKtr&m77 z93Bi=(TmX38_VY11eD&-*|OWj6{ zMU$3XG1JM1Lsl%D(jbP6iP%1VsuVKTGo4YCJa#!7wuv;VTPq`B|67lIej|ltEro{e zw%RW5*Xg61oqcza6b7kKE;(r@bd1y=NM8H4f%L8^u&C10^RcI@$()c)dnr)wV!({wa*$!4E8ic(blRu6&ObEV2O7GAB*F!*4H49H8T- ztT-MmZUq=_stke;{}hB&0Tx<1IoVY?V_?CzJ+v{8C-cV!tnr6AT!*Eqc1?HPcC}$2 zvD(u{d*dZ{G$1<0VOQS#uZil{`a-glw^SV6G36DIMCmz1a9}~_KVhg4uBg31&qE!! zVbuHv9p%F8`Z}v0%aGa3tPYd%=M zGa47HY@5XWlyO&kbPEat?)?MBy94_Vlq&@KX?`0gAphs?OlQ!{Q&Irwyv+K{UzE+L zKR>;(-^VEs{(Fq@Z!ZwJ4ev9TGx>eMuF4ZhF!_HC`LCyih;D(Q0{-WSga3Zo|1aMF zI`2jDxAu{{K}GMGe(`yELKff3cCURzM@>Mx0eH5+6vqH4pJgOpxi>UHkx)1RN^cT$ z?oNFQ_H5Aetmiv6sw0Mu`d@6lRal!*x3&v~V#VFv-QC?O6o+8J-MthmuEDh}?v&u} z5WKj%yIYa&tZ(ggUH{tqKbm<@axn8|&LPjZACEb=(CX|orf7FoE#D-8$>ky`Sa;mQ z??^(4gq3tNu4O>m-^?3h$ET<>ZD~y*muqgru%kcxKx)=l>?PE2)H7M|uAcv3T(5bV z-3N#wcd}Kf{=t;zyJQ58Ddm6x@bSZ^Own?CdWbX*5d_2u-|4iO&m82mnlLDHAUW0J zi%?w~;L^3E9h9(|>6UI3YxCqNta{$!Eu^-_PL{Oe!d1!i<*}+?mX;x6YC|kGH+j<% zV26|(Y&l5%Grl`pK8zgQHg@ZgCnIH`GprX`nHFQ4i)f>rB4)sZAm~7VM2P28mXS(q zC<^#70asalgAQNzWr6LFD?sqbh=Q@RkZ78CEnt!S{=N6dRo3j(;m5H?-wrRThCI`) zyx3%?F~K_qM#qLOnC&hibZ-e>mIR=0=vp|f9h30}p|=zgy|rh<^0Kwr&UbYlSW*kL zUW1P}Jk|xjuNoqu>W-HDQK?@2D3GC55BLk*ee5l66Hfo zqGwyRWHb$Ty3)QzCaO}sjTR9s=(IFohdo2Lqd%$}^DS9Z0}xNox9YPCt)5mYjGOZ6 z4z6@qAAgz!)TfV?T0q$29~ePRMqe|Gyc()GqW2_rdzTTGAEZD67#dt1-^_oKpivBK zpoiKdyXmrlc9huJ2K8UCm{E5Od?@>tM!%0{_7f`?-XK*r$2J3&mtKQ04L>3|o1?ha z;yGUy%_22;%asRT|A7O;?4V5*z2H?91CxHqQv93?FFS&WTcoQ-MSqyrQR)Wb#VJ~{ zXE*7BE&?Lv?)+aaL%I1rjJhGfkv%}dxML#46qh2VsgX-H<+SRxNUnCDr)YM zwTb{jfl@Dx3!c|2r-H5((B9Pj@w*>F=gz#VGhFYJCeT|+w|^;&*kaz&4x>fOW`6ij zPaPdv3HxKX3yHZ=xXyjp1?ArZ?^^jsQ7`S2&iC$~t)RL6nM@3W&X7sU8y?r^Tp2Zf zWhK0{p%^h_XlX%uN5~Eqp*NGX-e+etE^r+Y3XTB~sF4pS%67lt;4!DC!7%JI9l!(B-_143o&8Y`bYgw@LSjv5H$;k?mWJ)>V zj;y#wFUDMwVuu6lHXO*SXX@0%nv{6>Ip(b6sYCpN?GI-AN+GdxJu0-+Iki9>{3zB? z+KB1DEYm4_DW0n<3*gmf()C-_{aN+R=6k>J0x2d#k-C&6*N>JTH#0x8s&guDYqmft zH)}fkdqN09;rgQ;v@@Y}C#P&mkgkyx+m~INKNRMMnjIGphwf%ruZYCMf;Rxn80m!{ zK+zuzBhbmgNGju2vqS9319R$ai}){D}f9U0QfAtSUcrHuJ$+_8hbnyX-56}l6 zWF$@|sS?k;>6Q$Zt46tHDVWXU%;X7`EtFP6GauB*#-8~J)kb`8Qyu&ffeVC*#He>h zCnALAT6I0Y)LM*ZOcUs)%ikqZ6{&bK&3i$%aea({CuCM@S)vt7Ca>tm!`D+E77fz6a=VMbufRbB1vz;hbeF&=N_trGN+*`Oau!tx}+>w z0wdpX@a|-`C1*;+iHPM~9XtthwhKYilU-jB!}_?TXf6EreL!I1>4QL$B5{4rft|SH zV%AaaSCXcAQd|*zUG8x$zlN%`YR%3vO4uRVWSJTT9(HGLEG;c5{^^^<<6-9jo;U}5 z_1S3{ZqZBAN{N`9eoM<_MXFX#)a)D5V1lfjKtN_VZuaPs zAke;KBCn#{!ShS;qgI2~V~yD9TbN1W1@t7maU|gNJ}Q*FNi-sJ--)-MP@vhriLtSI zCbu5t@~K9O%7n;jad*(s;I{x8qQ@;(3`Ik>9|~B*UP$KvHH7r*FFl#+P*`$AhJ}Yx z;6IqaW@06wR7HAhJZ~U>y!KvZ`>mBWDFzLs9|0I}!_wmAAtg-+CKT$_c_?u-6tMpu zU67V;bn7(&jm!lORrg%&WTB`r4I571cC;atZUo)wNU*9SRoz23ak~w#rLo9k-w6N1 zrk#K9YD5C7=Eqd<@yi*<;8-)kOI{uKdQk86TkJq91Ji_ioC@vmEdtWeL&Yrh2#ShM zB0-c@TrE~Anh=qzjwZ&OTo%mK8-QW=lVIIw{7PV+&r{pOmVjsD4Dh$UxI|562GPF0 zyiWRFG=gcb5cMiCFq$*VovRKDRU>VG#+||P;v+p(=vX~D?pIQF$t-Or`xJ#=C5t+rWFrhk;a$*`tSn z%lg@)yT4+2jm^+?_P;)VCC~N`>gG0__^;O#xK=CjeW%ku@$l7!B1yysc|19HwHA|0 z=!1Y4t1IIWDtOL-^jZ1Mts98ke<1;Lax;*|QGc(O6~XppY+I3Zow%;bZ;UccEd!>1T!_w4HTE4!Ge%^FJBa4Z82mfkM@8cv4FfZ~-We@#y%R`PH*Qf5PMQSL11B7{!!Kx?We%rkm`29qT;ct`|MfLlcytQz%hngws#=}q(*1#C zT&h0M`6E@3=m!J&Y1fekrdL|y-_u-fM|JzlYPzh1;mKpJ=$j~tLRym9$sJo;5Q&WY z>wwW`Q9ZTw9%CM=35E42YCm>`82d?8t46nm5P)kgL|fovg$8Stc15IIJWs$?e$l~> zBT-jHn)@;yYjC3f?nd_#&pVE=-|JSZr*mYR?lOS)RK|l7VLCGbXx!RiPQRrW8Eq^s zsfU8W6$cf28I4da6=kidn*j#V))|atwf}|@|5nTVJEBc}eqtfRY5`Ay7kKwep(YAM zecZnTPa*gp^>(HIMP9pS=L!4%SlV2t)uQ3cypGX@rbA=$=>qq7im zQb^*Ss}T+%1ZIFRNKkb}kiozMcTb;5{~@;M&r@=}V6KJkV2EsPlO0uJ1be8rIO)7+ z_rTPkX$R6=mnK=F{b$h#c--Kg;QFP>!BfhXy}8dj=)@RlA4mkoHHDI{5m$YGbuY9k(CCWP!cjE`vR6QsbqtH7PZfJf}npmppr4}uFtEATH zN)0f>m*c=)lnH05yw+*{!zbk7Se2&@sr$UivG+MvtGNC*921d4)01dgUD!>+A3 zLRQEL63dHE^yn6b3I`fkJ?5EnMTVa!D7-=kBg}MxND{Q{r}IR#XCv;iH67*6w2A_j zOgZU0Sq}ml`SN6?<07QFS(7NGE)bRylr{djH!(s2VG0xQM9P}rocTLTy(M`3L)NZ6JN zFJ+^SwNzL;jt=u$-m#!_HPn+M9^d)8pt8|5X4L0WQGb!r@_ohsg)EV3hA0J+zF>}; z#wabyHNrgX_AUyUZk1iJfJ#z~Ga+}jNRP58IDYJ(FTi}(SrBUyN1W4@^zqdth3mSQz>xg zlwPNDY3TN)n=o9f5TC0EF|0RH{<=iKsShR@WGg`o&IwX-etW8mP?a4iDJ`p6CIBE#}6 z?BJBwfACbtMQT&E9wz&E6c}m~G6Y0g z)V8xV=CVR+h^8hzmYJNj9C4Ge(WK_3DTzj1sk71DETt?)|C*||9`jyQfz410#jV1J zJk?LRQ7U~Ko#`ct@WF=1ZKQI=qs@9tRo`72`+bm#i8E^nEk$ROfB6>T$j};K&}!__ z068JrpMfdZ$XGM#Y;u{`g)7Vc73E*&p~;08t5l`ug7>ku;ZN`aLRyzjXWIn@;Joo0 zx;^V)K7&AxzPb%}T*bH|aOQxOl6JQ*M@~91C5DIGai9-O+{P@`bEsjTL%Y&Y_|3U< zFm>h;@^$4c{~7M;YZ$a%^;$=ev%vQwDeb66ABd48I#dVAH;bUiX(HbC-7?S)zGKYf zltV7HnpjF=?$IPb#d32Gs&|m9Rz6KFX01kiPyDlG9~tukUXI-0tM8hbklV6uG9``s zLdM^^>Ju%CKIV`|P53cFksGi02%v|gt7p0VYMzo_7?n(>i7#d`az55vxY#9gv3T|f z!G%H=9x&brpsun(oqpD&Le(NLFa0Du7DK*1l&vG&HY)p3cW8vuB>N?GkhjjEXPvbM zGh&*HlwP~tvgfR?8I&9Mi>P8gXw$k~ePg#yQk52!YY!lpc|0SZ3X_uo_6SS04L;X>E@C5ipeneo_V6 z{eAh-*c!qZ82W0+p$`Ds%wBpJui{Au`LI~DN|t?VCoqtYQ6w7%MyP$ZXJ*`o9j`bZ zj4Il$9y{jyk8Fub07CP%p22wC$x`((7=)Uj&Bq;SVLKkR>MyR0UG+Y5B$$;zTwxEU zbtHzX?GTanX5?QSqdwk22RZ2`(2u1R-i!~;o~6y6JtM(Oz7G02TMyhGtN30mekA?y zeK<9##9LJgp^34AYCz{jBuexFPp!gXrDDF3+0^ToG(}+KdhPOMk_6(C#GUC_;ZwuI zakE(XPG>ii44oCBoqkH!a$-ObX8G=!9Bl54&oQVpL@DwLA} zH*Pp`gHpr4IOYC#9~vh!);Y_Y!=5G5b$wUN>fy%B#Xt)kTSs+Y!)?;|Is~dB*7+%U&aMzB` zhSI%4VGL&F;|#0X2r{^j5g_`N-g|gXDNPW^4d2zOdb)`hqLgAGJ-X4ns=e^g&q_A= zCChrBgk%|E-$$pXmyTZ6SZpCry5q8m>Mzxf)54oy7nL)T7v?ioHny7XKvNSeIK4c@ z_s$5IQt;;e>@Iw7eOzM39pR;ViLA-k#y!Y5riVw}amjvms%rJrd*hz+~VYm{(0Vw*=Zicv76p3X1REhHJ z*`L!^Ma#Pdd?}gvZW<6@GL`D$&VDY<@im&s-M%XHh2V-bc$1cVz@ypl&AoN3ZPVA> z56E<&nBEQ#%R;N7G`YLIPTK+oCy#HTpe7dB5x_2Ca%MG;D=#sJnrdV) z|2klZ=SG)3u@;p_Mo${cvUsZ-BI2N`r6AlZrh0ly;F(BPxW!55t&<~CfWA7{zXp&u zZ7MlC+O5_%Vhvf_q!cv%_kVTJxB8*H5s+*apC0X06y+_S>tNhXx`e~7ZT81KD-*TX zT7k_r+V&^v$Ds=^bT4FsO!q$dgw7*r)g>(J!k))ernGKey{qt{`lomNP0?I$3M#Bb zchN(b@ZfQzIQMbXff8||ET&LhH<+}Q)UNgkWA7wDJ%BX_Dm&!h5YyHD0H4I?h~H}zrb%c|HWF3#>>@XvxO8iPXZ>k=#ASIRarm--7jQ@ADpt`jBGm5_)|6oqu z{BIWWSpYMa^iPhuLGxnW5VZOLPG9Ys-hOVo;JT-?1?~eOp~0oCTjtJU#!tE#!An|?`1x7 z!#R>e7O*gW*Nf&7vIfcVr3j~JaF+-W!8bPaFl~LH-qsqfgpk$D*%wdXu?|WAh#_Q0 zgcaXKp>gNkoA~YDH;mAh7sS8f7;hO+>(+Zejtz&WVXDT>JlpX}DlqP~)VgHME|w5V zA?X6+EtTx8ykDB4c@tsNST+L{5lQyoLQJa&+2=&Px)AzfeNH}NoqNWG0 zt~^ish2_01Gn(a3?0u?jnP{X15qIs_joBxYExMRUft1ahrMx7l1Y zSYCOsf!wnl{$?ROGH3g&8!>UO{m-k#(z@5bjNW@+{d2nf0o;K{Hfm7F_py(a8y3MS zMq-!-dPRS+z|PzPvE?IDQ9YFDz+?O>sfpxnDc(!BhSpShOa=nWeq@#K{+JS;7+~0T zeTM%j&c1TguuGMMz&C<_b>yE}$vS99C#Ji=pwXc;_M^=4$d-X5w}(Rp0zwYY*? z_x}lz+A*X6!%qB5_mSMp38`0%V%!y>B->t56(RSAEZL8{HRQyKNMD84u+)<}5ot`cauba;qdo%vV=a18#9 zZ#0O!?*}?X!-66U9t_B$6fkyamfErQj63?I@O%*BhOsDwORgK2&|CySPRPDY2I0Q0r98ITF)wuNYH)?Q2`aX*s|r2d z0Hqs!Z7gP$+sTRpqmH$mo8`)xgK@6TBoQT)z=64Di|A6luu6^1T)?AaA|@8rd(CLl zY~^5ukNR!tHBZsFvyPy2>VrEOWAgu<1^)LFxe-loi@Jr?UIHMp&ay&UVgBup7|rlB zO@!&+99cr%`@Zl2;lR^*{YJ-$?&jKW4ZO}_o}Dp_rB>amSObI^6d>bq3<6-8QY5Pm z;a-(I4l(qC;8+k3Lo;h%ApHR+O7riuf&xSqB} zpbdAGJo#TY%mpvcjNew4K<|77|6p|BKFhb*bN$^ zJIajqtntn57k;)6;y3*@8V{nUAXxOK2*^C#(=dCM%}1!{{QRuCl`!W0lf7uvtIfRH z50Mgy{SXsZkIEopceWhWR!WICf?lU5*9XPpQ&8MzA)elmBxgxvk&igUS4kEbaJK$) zfx|vSVY_{QF3$?Nf}DvGbFf@V|JtFti?O3Wh_HOX*Px)Y=g0a;g+K7e=Rv?VrGd?# z6NGLc+=!_5$34FCXEBazjEaGZw)$B0OTHjE&EMt5?z5miiO@`nYYd4}_I*~(3VjkP zYoh)0ttTKHFbn=b)6-jYf7W)57#*)yi$5YxeyM8TfiTXZI8F}P9Lo2a`y)l-bmuRY zk({&kILNm22$?eLW#o~x`ZBw9YWuw?!y z<^-cKyG)yoSqe9V34+6r;i(}^|K)yun;}8#&mN`uhuMmu{INc%rfeT~@U9`UK^lIG zuUZ5l<#Fwp7RJF!a6IFHgp$EkE>)lf_(;`WQK@B4&;1zdNSvGNv?#Oa_;i@5LCxrM zosY9_qcCGzv*|0>8bSE40|Uh!eIIIc9{psR!sQ{+G?wF?|ytO7mE9DqPCA~l?d5+@*izItNKM6h1Z z@dEIvrmd`6+rB`jtr2_*Z7@wp81Xm1B`zskeSv$YI&609*`MfL8E5+0R+sjWBfzLF zPu5!}PaB&UF|5t+_BZ@fR7Br)vLx!sd`8<$tKHI`DBi$ESWL7ppz>MiX%3M5rO>wh zva8`A%sC6trLT+Q-60oTn{qi|+G+X*S1Xa1fOH`Kiss?Za>=6GuH=*Hy1}`yazbx7 z6kncVY!QMIog!1xa=cPkR(Te|Z-o~oIlvE!0(D`KfaEoTEtI!iEaYe+=$srFtPm5N zI&ax~i0|vNz98+WiRIWl<+G=P!fVqg_V3(Kw1il_Mho`&qrc+~fN#!2Gy;*l0&8au?y zk`F6ie+;x;rtuLw)JpT<7U_y7ST1DQvG$C6{?(THsjnE8%q3VtY5+6CGi4aA|MX~C zfwjA!9`)&wVx2mSR;5jY%9w0+ZWGVEdC6Mm_wy8;oYP3+&rmCg@=PRa22T?ud}KBj z5jyltE=CJAju=Im@Mt*f`&nE+wZpt(MhY zCt53Ic`Rq;R$s#|B-HC%bPHcZy(bNlToE)QIWeojOvsVQSN)v;u(#uupME9;wGz}-&U-SHaAJg-ZWloZ75p*!D#dz|GoeF z>s{<|;m2FCqtlb#;6IoNk96;2nwE3Ra~_m^2K_Kpjw;Yw5bMxcUf8r!mhYLr1b>&T zGzD+$7*LZjHAu2gPjN1Ph6eP3UXk=lIw~2vW!UW*K8(!S6t2${YA?T1`hZwP@zT!M zbFUVB^E6F+D=+T?e{zQ2FS9%XNKCYTZm9W$ki55DSfv!hy{Jj^gl$EM0LiFa_m#Jq zstQ;Wk*~0I#kLpzz*8@rf8#Vu*0FkI4K^yExmy8b{n4dMxPccOu<)PM~@xEaktUmMd!B6MbpIUzC6{0BqV{p&{pOD#ugpvpg(NV=XC2B?C? z0&SIGC}-Jb3`%#B5Uh%yggN~7FnC?^U~uyfW-W1&g~lWJ|4h#Qw{rZypF%d@)L-p7 z?Owa@Hg98wMOP0q!@NYUOYTdd-!c1bl7;lvB_A%ib7j-Qm!e8dD96z`C>9U$D<;_` z=4VTi_ax084ahl-Id8MRdsVx*-L;`3%BZq zNXlI%o>cI`At0y1U(gb@#cavW$G*kj}K?r%k+ z`hGy*c`}rP9VL~rAZt!zcUZihvT~Co(E=$TT;WHIEEq#gGS&8Q9CN>4>9u-M3!9@N zyNeqx7uK&+e2+U z-M2rOQaRn4%)Qa{`RnHxSxuhVWy^mr+{J^xQvKDqQCRo~)BX=;#hvKhZ;=#(cl_BO zu68Ms;n29uT>pWmKM5IX`ic0NP<3W|`?sq`5lC7w5@q_?JqC;|v>LvIYI%O!K3jlh zRMGfgU?`h^8$0BbK;%SpTH||hV0$-)Mx`f$`@p(pBb`8dbkj&X z8u2+`qmJd;vUXD4ORTmJ?6sD+$_?>~2Bc#wGs%@N>v(s>2En^MzexCx46KS*Gkf4z zXC^S@()w=&NV zE5Uq+nXDn~K@ig?$y?PH2EJcuYrDM~<^K4ixx2l-?(Ej;q$*=8E#eSl9a_cEhS;Mk z1Fn)P8o5%5){RToAL^eS4Ij+oEuY)j4nXouiEjFE0@C_zak7^LmXn6xsKnxx2EN;3 z$%GJbV+Vsu(Xg3U2J{o=IUv172CFgyfKp zYZrf>|4VnQY%f4m$ghecs&-&1QT)C2@JV|!#LRSIqWT{U@B!-;x91;Bl;2S@Ary|6 zU@VGGh8G^~c`vR0eE`@;1I$geOAa z8=@LNaZBAJya;YJ`mpm=D*FuA2S-_{7kQtc)r(YeZLN>@x+gk-^UqF8zM09NUiAiV zx~x!_OhKLixs3w4dXg3P%eUiineVX6@fm}J7%V5ai>AP3do6AR21f3&sQriD5_g;G z0hy(&$moHqoA+H3;q#vv*7BW{EmmaBzAYJW=f3;-fh1#hXU+tp8D`xq+!ui>baz*F z&7-v@-&pF<5eo8-AFC_!jPnKrS?C>XPP@hpkd*zbTguuB3NfF^c`HYZMd_vJc+w>g z&GVHkwSwJ!_nf&%5hSxe^o=Qx#tdXvW7>(-{^fv9sQ-4((#UX`Nb>i` z^Q`r^|6GMHUVj^uw;aFEGWfKpw6x%Vp~D!OaimFi{Y8ZfSt@&uJs39_qRU7xVmF4P z1jozEky^WNF%XGimxU-u<6s~24Y%TMDS=!8*@3HGe@W6x-Bj6%apD$Zfnb7ADnHvM zN0%7mJu20&p?GNm!$ntnKL^Z|^t^>!KPJyKDdHiz9)e+}3)z1te028-+k6o{OMHP_ zsJ)(a+sAk!j=8VfFAr%RcD!CF-$OIZa&QyvyF4%=QpXt?IpEv}e;G__5<^dl#)xsd@D(tf@Q&iy#Q zQIIv6b%HL#%9jBlwpjZbNeEVwb+0liT!W2u}+CR7!TOx~R+9 zY3aFDkU)|sPV{?!e;T_9)Ch66$bJk?z@fBVz!*S4#rHo-)*9q0Vh!Q8G^q~AsD(&) zwt4RB5V(RfDk?HIy%zQE{+yY|yUh=93@_cFee7tgc47$>rv1R>CSss7 zXkaM46}B{gUd?8_7S<4zBDW#Cb{Ixl{JXy4|Ko9(A7J)xL1ovu}`wF_ZhsvvrPmG^YO2D8RUw@6vU8O*%<@3S`H_lCO$6< z724`1_UxdQyp0LwO(*th&hPt29hAt)HQt+)_dyQ@Yy&M(c?Bou-#n^m0*6!=9v|u+ zrgRHeR}Fvx>(d-At)E!sD*^ybtCG`3&|Pq^8NVd|d7O>qL!w!yUkg{tr*p$4Y-y)L z$wJX#6>aifXvAie%bsJTs~%G=TNcmJ6+NxIsl2LggtdODIjkIqK=V5u=&wD3ARWQ2 zLY>&C5RTNUR!CpIR^7y)vL z6SzJkcWMNSv_~zZ9eu5t@#NXVU*nW$wFG~DYja*Ovt@_p4KW}g22s#pyLQk^;%j22 zL#Pjq{ca0hmAY2bRbOp532topRMo6Cuys-BN!dD}?%ijy?Q(E*Z@AZ-!>h<^; z=i@b)3mzE(oh33+Z1%M5u6f9@r0?l}XX!^vrQZ>z#3bxo46?UN5(_&0iDD0P@iO9- zYap$e(@#U1$pv#!Wh{;p2ACRzMHKo)6va-_Iay`R{2nO3Iq3+fO^hqNuOh4L! zuI0#bxGS0TfM7=wIwuhjeE^1xw;o1g^se+B{Z0{8-2%vwf>(T8kbzGp7D0~ACXd${ zmneSA!-Zevm&yQpPmJ&U>PngZFUyra1ZLZuFXy;Tl4JU}^lAOS5{=KKqKygR zB}jn{a@k4T0NZ7vFGHos=qTG;ZtQ*a$z+NX)qaVlapl}HFP+nOgqZ$=*a!8PePs=v z?UvE;Z!THJiLHA$aw@F8W9zVI6#kVljFC0mwudWktzd!W-=(lqRFhdRD5v!NJemBT z82a51I1|~0%A-5ZW0FjJv786Z#EdeWt`!R8 zJX5n>A$*UOdYO~Q9?SaG3p*kV;OAH*!wh@LZy!oGmA}vho+cbk=%jSY&v_n1ty>Z3 zEq_nw3d+#fbX3uoK#J>rCJ-s}JNI29`|7Igr|OY^UoR**%Hw-8I5>LUB{e)lkFJnq z|8RQp`>>CkOLLW5QK-0Cpt`c)V1PjW#j*D`W9{fcg~o2Dc@3ut4Cj<5XStdl4j6N< zopvLU&yEdKo5p0J@2o&nj!?$okR4vznDT^leLK1@b;kQFPE`6?obJq;dq_h>Ex&KQ z0hH%>Le64))t>C8fP~|jJzZC$zD0!9j0#(B)lXDoA?aCAy;AB2N4vgOIe$n~NZu(d zyJ)asG$!V7ILE)r(uB#)M+4zu%|RHsdHq7frAN?$h9gb!arg|#N_Wzv9;u?uvNJVn zwxrk27E#f1=*M|8{Z1(x)p&s|E>wqDJ)F8&wJ7cQ*|SX*GM%(FNi=(Bh@EQSW#LRC?iTBi4$732|F_VDTF#?3rxmd(?5N?Y_*&k@__xu zpLqP%Wsoz{kVPd%a_In-G^2W!i4t<@(X6*vZfll2d_#`l7L&_i1ZC8*lf7)qt^mA@ zx{2>s36v`4M@LK(HQ-ExqVH5n-vChuaght;Ccp*+D{Ow(@vOWe0|c5BFsUPu>sJEW zKow-5;Io@1p~d7|g~fIXVY@UAj)Q=SvcKXA5jq8CbpDWk=5;bx^RlyCT|mvHriXD+ zr60e1pYNEkg|Fym(ac>VBy7QO;V+Sgmq6L#Pt|VyAC|Knnfo+I*&&>5f|@9U1mV-v zOb?$LdS`>~xj1;&elb{_d3F}&VUgr;zGzOZfg6?i0jCIq$^w?h+O>cR_#XwK(IwY| z74wOn(IBoCBQ4oF#9}1DrK)MUO)Ss$#n@@oSQ_VeeD{4he7%(=L>I3I@ z238F}ZMhZ%w1ar-z77xZuoS^?#TSC0v!8JxI3xOo3I+Q7lN9IW|$F5;7SEu`}uyv!v&5p+iqGtZVIJi&rou@b5>1vHN({E|paeSIA z{{khij|EJ2rZZVRQq~yur7Af%wPhHC(qxP^4WrAIb;nL(Mp#rF3`>eL{<0VCooz&W zT`ANe%jX}A2fz#&iY0KBR#49@R`2xxhD4?4VSL=vA8*NO+SNapP4|%>aR(pXohW8s0vi>S4FU5Pl@8j)K@Ha1 z^#h+6R7_`SZDzQi7L@WVKU|+~a8Y2VEA37nwKQ#Qog;Nxkwsy|hhz5j4JMfMZjT7< zMsStxzjTl&fj(zg_NG!kT%#_XWbudAdUy6`gJ^fnR+1>!<41MC9Hx!9T=m0t#r7AU zNqg~9JNcuG%v7u`uij5=*sYir)X2Zsa<1Bqf4>8@jG`y(j$2q`(e*_G0m zl};7}tVL}Pl-SR251nz<5UPm_DlYK520-dB7fmY<Z_cDLTGw=O~ zF{mggD00264>q#sd#7NzwJUv;TWlW=2(Laoid%g|m{`S+;Vz^zCN+?Uw9#6v<~X); z{-g&z^*p4tN`Fw@*i8~%jW0sY0TFOb4VcISy2xLRq<4PQhsBv0I&X)_*J9VTkHR(5 z8f$6;raO;KmGu{qp5)Au?NPqf88xO_N0)KssmpUA2*&~c^73eR(&FIoXW&2tB#12L zD;yX7;E0d4&kDx)<@cFGee!lgG+q2W<|WR0JHLI;rFdQSn=(l3llh`v721O1UcUhi zER=QnS&Pv^4)9FaWNQj^OO$zxxoq!hOqb?WPFT|(pq0%gfR~s6pgjTMGfC`@%L;uR z5*^zxm06AvI{w(=M(uZ%&Z^$V7@R`dY%N4)3lwR!71oR2<#ZneXvJcMIlN3NwK?|f z_T8AY@g>!n_~P@TM*0O?fBD{}e%XqMXl+WZ)(qqXq2=>yH7#VU+C!Gls@hi1@0Pt9 zYyQ4QnM}25&P>jjS!G&fe$l75SNy6Kex?dFgo{k@o@mBdhW_d@4kVK7+H z$z+%{kNxLifl2{yJTMZ8TC0#D5&fTbEVO}r`Cb~tyV3A&`q$-o5SZ_D zJ1*6h_cZH2`~2;=`w%a|uXze~66I{nu%X|+VT^P4>r<7lcxxO&!;Tb4jTDH@^nF4; zN4U(m87W6xY9g7bNcXWulkno8NI_9Qh3R8TaJoz}NHBWieqSkYG8)T#?w!|uAZ)=Y z-uZx<*t}j(rCIX#ULAQ4p`stHrq%!y$kUQ+g=2p*|M_>+^kubrf&6|``I+R8hyww!X8LOdAAqP zTyE3i98`C#S!P&`ZK1Ru60U!IYoz75(<|d-V01KCjr}QP4}I_N??Dd(aUn>+ZuPX% z0Bb+wwS@Nz6*G!JW<&~)PZs#QlXJ=7ZeR~vQQtnYpI%a zapq}2wMSm3ST#!-N1r30()2rO^=eYy&-ZbGx;VYvDxgVq-FV)g@%tokvtjwX?3cDb z(_g?EWCeF{Sf_d*ED2dXN zd)y;b2NZQl?opA|h$Jp=M!8~l-wpeU33?PIo8E>JOW@Mi(q zreK_MkR_+5vX1`sKvIici~=3ktQYzIAOZ>DO<9NhQlP7NfdmB{HViro9uhqtVO$gm zMf@-fx-}_oAd)Yzzg zT0V+6B^-RsNgeKqaK=oBUUwJD*x19+v&drUzuQwsAPgPl+VCYVLtJ z45QbmX6I8--81nd)T?g1&{mDz#X;s!2UE2@q6!mb4YF++oxpaSv%gEWkYQ$oLAN~q zUG*^XMs>eZ?*W9n? z+2Uia9d^Z7^}Xv=gd*}L3q52T*%)r;V^2eDwIbn1r!PUvBIk0@iI9?Vz7kHP9r&TORiY_FF*^gcfa00{RdO6bggo-s_AhCXWqy5 zyR|?gF2;a5O16G&mrR9gmdQwXrHtvz(wYXA6e$4*VSt4d47GOs?+F#^rIZl%b+ARH z1FKt4RL~!P`RT5ew4p<9ZPptmkej`8_^<~$t`JG2 zN=5|Hj;|w%(f4t!k%Wm2SyNfcJ0cs#_ZlypE~eWyP={Rp4WrTocrQnU$oof=j-qT} zdR>jg&|kj9I#3uR`Qd_s+>WbTsxgio)Y@2u^)7RHi+fZktx}xzsVYH+$yc?id@cDb zc)!>#>pwLMikAxmp6lEm(}}{r-&|5U_j?qhHnQCnTBkOC-@h)0MReUWs5IVBPZrg5 zT7`Anc1`J99UASY-r|*p27vbRMjyjYTzT3bTz_EMpODhfFB-fBL z7v9rcPuCq?ZM#jI;B5U>O}nWXmRXZJ?<0}*8qaa_5iu!DqPYNtHp1;_ZscOSVW2@QArYFZ2C;q8Ep3@?S6qxqsEmH6%G zp6u*Gi11jRJ;N>_c|cjS9O2W(_v5vCh6!!W`k@NlJu(4JSpx7~ZQIH@M~_FW&!%K! zB^cl@QJo`$?M3-LHSo_}kTmS96wa9oH*rq5oVwdw^3qOuaLRAT>)TIhuDnU0?P$D2 zh-_wJIxX9#om)>?b4TmRj|a1j?!HNkcc+dY?Ct(b^ZB(`dMBSMlikNK;f{~(=0lXS@B z`K8q61K{U!p_a5-^6-saF%7n2T>P%EIDsV?j+Trw5sg2svEVqIT~)=c7{GhiZaXoW zK(gu{J+m^BNM!@zaJh|0@B;kKIt3J=?g#-Y#{=MC+1)I#Y}JYH^<{^SN2**>O_7lmoc*xh z5dl{4tl8Fqeb3POA(*NaZg)aSx>xNGX81eDqu<;tquRyA>y(NfLr%5!g`OEw6C<1M zI>900ByaV?%v^QR2<6e(TVb>)crNKhgQTdW^{FX=Qn0DRtQB(BYBf5esYg z=~41!Ij8ePR`MC2F_g;=69ILKSqJT&l?=-WM4_G3IGGFPY_by67I|?SJX2bAFDf5GK}~kZK3Im zX%^9zy`jL}EM`D-UC}Oy5k=NUFM~6XnvUuQc|VDhtA0O>(2-~?-DP;`t>EtA|Mqhi zels%o>(@y9sZ4|{)fVSoenJqAWKXDZx_1;Ap(tH}%=!u3kHCCgLPGo+V&oZi5n=A zno-T%U&0ksHzCL-dq%KlbC0~rjMV{pwWRYr(BUHQPA9_`-KAP1zYSPmPQt_v$nu7v^wtqd!0l6B0Tm2tXx*5)^s{`-;AI8oqDz0ct z*8~Xe?(Xg`!JXjlP`G>05Zv9}-3oV?!h*X5cY;fh+&ZWG-qZJVKlDrOG3sfLU3;yX zYtH}w{zwKkF@YBQ;HvgsSR!YldeTIhuWK@`(=Nlmt__a3@us{{99>n> zye$mvmMX!tw=a^gX}%(oa!EfSGkI@pEVNcG>FIm;ljjuHBVVl%;4#fo#cLvk+A6)e zzz`iqvW0(^DmvB3z;I@`aR0L#En;JSF4p*6 zEH|^=rIV1&=KmMMVqV)Ej6FL50#s}&hgI@|VH%V}xEE2iGl?Oxjl+oTg{)&i6;llEIPE4rbY=86_e9k(JJ_L(A}-K=8M%CrVF7ZWLWr+q z2RH?}YzwXUjU+qgFsBIO%g?|O=*-GuHH)i7P6hdHHKgMIaspLV-^`x=O2J5|4xW=`Rid> zI=I9n%@7}pM+`3m{xc;Gp)y#+jsk!27eemhLSLelR!PK>b@VqUm!rUU-C7tu({_F? z9A7L@qb7VzRR!HO<{A@|${M7Sw$%HlZgTI#q(D2n%bC_*qOGn!0gwW@Cx5`qAIbE4 z(|2L)*SuUczF!)?qfZ675ocYr$A(esx(A0xA)#0b zD6~0DV7C0y=#?aHiui*VIic}UOg71NpaW)Dxc1dT%|+|P*kgG`ZiWwk_+fXCzYnj? z&Et;MenxD-dFpEz(4LnS(=N6x3W2(~jp*n8iwru`Qs-c`DIwrh2$B9FPput zEGDG$h0o+&A<*mKZTaJ?ykP@q~*As<=`4`(G{lmso#c(FRo3W zeN*6H2%^}icS_g$ZA#&|E1Xp`hxHLb&WuCd)3P?o=(6@1k2Dn0oI}aFiO0i{64gu= z@q~200Bt;ySCV*t0!J4wz<_U!fx0=bOcDehn&HcHxj6&lm&G= zf)x7Axkke;n9gojzN6nhT*uH3;CO}s-HBQIt>1;E3}4%Dtb|k3JE_?*?tQHw4@@*I zjt<^UbTuZR{oeEomg4`{dbNNtl72vkniuM51KUgqr`$yXmpl`CYHAONvpm0m%T7V` zBjr+cGUUC*gVC0Gt|$(-cUZmmHIR5x&pY&tP5ApoL!4C_yV$d^+GxnFtY^KBpf2!k z;b+z`#f)B`<6);_6l8mKQ*5%Ed5K}c+@LmZt!Q;KYvlr|R>po+u2}3InnM`gl)8JE zq9-g>2|Em+%&ioEM+V;j>XO%;Z=lU}@nU8>6Ay_(hK1SA0HK z?kz^Upx@zm1YN&|M-#jx`sf_9pN3eSUV8fa@#@Jhl<6lM?0wiLy?qss`bIeTtfgmH z*crkbZ|l+Gu$z79^)cYv9Jkz(Y#@1gTaT)`+=R9na~Vi>5sQ+&Y=o*gPDkU+5c6rus|~-0w(VE} zU9sPSxU#vNe(S6ye>#6v9pQ+wDtIA+b`I{!q-Sm-$tpaab3tJQjO)1%X)Jz`cfxX9 z8tT;9lDg>8ICT!B3I7KDP`{dHy~ zpnQ(SYJ^pcWT8H*)pF88-mX4}=gk%YgwFLz_at+JTbRI6-4*^vsFC%x^ua@eAU@Dg zEetn+J<$|UAN)gkjG>d8DoIE;VpNtxPAihz`MxRjM96%jX7MSA*s$2+XXLBkV#jjZ zK}r@9&uM45i^a$8i7;&(kP_!?mMSQALSL%ZvgJv`Wwe4J)fnFOkgz9pQF+_!Fqds{ z-|&t{ZGfK%FaU%h&y#0A=m(tf=+zvci(ds}NJlj}Usk2fO)8u_OmFeYw=U-m*qy#> zdi_XJpqSl#U}i}g+bgQ{$E@ThPwXF?lV4bnJiE?LtF836kzY>pwF1`CpO*4xlxH|% z4ny+hBVWc5jpOZ^Oj_S4%<5wK(b$g~sivv+%gdN6TUc(y_`P|5N@B8O8QJ~R?BGLk zf)Oeo$+EGmZ8(`wk^V0a*#D+8JP!EnTVrEv*GZsm#J+iAfk`*Wq3FeLj$J~x>7Bfw$chJ@#~XCrG5;0%CPwDb5SElg+$f!bS)M>|W;c_AlY{zhYQ{FaR z;P6wRk~M49AC|2#y_l>C3;WRWPN{|2w;+PPnSyC_1!=YFtAJf%R z#GXrp6VfFwH3;h>L2H*k&|Qxn2{U+qn8MO}Z1Fy-^mWb3yy-7_iqDVzSoTeYJ(#o-=34cHDH3sP@)5Yka>7F4g%1ZT@{$Ng> zj_q{4h%Nxw0*O$h5lb$JC zqC%m_IwuSc6V&Z?++a&G%bF5DR@SSVa7vpA_P3i`r1k|B7L=Rj7=~nsIIQ# z(kf}_XjHp!V>#AeE)2`wfzy8>-duSY?_>Mt{z8nwmMAVPPVt&iZx9NGXH8}!)UY8| zI0}0Ix)7(v8bhJB3--6j`9Awofk+|$hcE0pEso4&yS1)_ zmW7I@Jn{tVNYND>%HzeQ$T=H>5yzTs4(ioOAk96rv^{w0fNzfCvmyr}q6<^Sa2XMR z-uyF1@Eiiv|-8Fn(?{>cOB6D|4K%#8G4@Nh$HkV>n zYCoAmC6S=n4;m}pmEpA=sAb3=l5!N2P)k;W=cj~~4K4NwztPybG~4k*ZxYlL9lVM0 zt)DMk(@;m!Wu_NOQnXYpk4^i8?Yvr|#WhQ(9aYr?9QE%x4_OEHWJ{;G-q6xnvriQY zP+UO)EncOUm~2Eg4TFB-?Fb)KJYtQjFh@+TG4e=h#wdbXlm;?RKo<=?6v--eSuFtw znxv{hSdMZoRA*{j8mkmOJ=aA4p`rQjmvMqy?S9>83!m?6MBjori^!KmbRiay<+62nqjBla1|$G#NcgXlJ4J+ zY)}c}DPF`_8%kNN__MN9?P$F6PpzdT{Sj`>w|5Z`#mfuRR}OF(tu*4;?5g`STMG&l zJ;M)1vQBkUz3T_RUvj3AxI{-WF>NGWM~)6YB4Z9rs&)*Xw8(&k5osvIXjgJ|8d(ku zDt$PE*o9lk@eZYI7^JKso`;++?^BEHRxw6LRR9=}$=rEfDgWDe4C=d2)i6KdCy$O2 z9etKjgCP!2&gCg>v|P#UPNIhcPk9P9PzO88W~BFb3gu_h$Q{>Ii?1V@E=@x+oNQkLjNmv8Qwu3Yc?lk0ex-WB$nhIBa~$AmfNhuO-DrB%y# z7%BXqY%p$ybD+RQ>nBXS8+oeiuTkkxl&w>*;89+K;KMolsQA1VCn>Df zC}sfT=^^3!>&(J?F!((GEI!`%7M@mpe%=jk&J*hbu0Yh!+j8+6=Y&KVL#AX9Jr!HE zhZ$L)NI)Ef>_f+n*$mF;5zRoMXO7Swv2&St!w6rg6hgOS(?t9l)(rD7M49-#ni#wj zim$;0X!6)pmXfNbaj6!Pc8}4qjh6K~-C~wA*|mVEf=+pw($*aZ8wIWB?{*}CZfluL z4+csc=HWMvIOC0#Q_uaSpMD@x4xdb}(|sIy>MhsxwANm> zFs#WdiArzFA9Sg#zOd9`H>TdI~$IYZAj!Z;H(V77V=A*`KgKt8Z#5Q=z zH`!Yztw;ujtl(+=LA55Tzk8XrSG(0Wc&?ZQhm!lJZhO7Q*P4hhWigW>X}sI0rs8qK z6(R6o#l?5xs6yj11?6Zx{LoGo*o$jn#>La{W_YiSitM+?z;K@6hE~+^<}1u1m9ULwV| z4E$PEJ9;XtJyTn6b1m5-xLeasYhT){jJ#m+m%Q^(?GpN9+kd%pG)pjBWu9i3T1XjP zCs1fhTB+WdrgDu0<1-q2Uc_}Wbn7Wc1WeBUWRGkebjZW(-+--jj$lINR5ow#DCxqU zI4CbEC{r(%*mquWl$7(=4em!Ep;)TyEZL>4CE%+*`q+@J;?;WUMnan(`6le4E=A-f zqzPXARLRQZWMHVL+LrG_TwgeMT7tV^g#bmQ{D1N62!SU`uL$5S31M1C3AYJ zErd0Tcr9N=*U3F8GUhtiFE5qWHr8xc3m2!<}B z-p=KsOJddYWbKj%_YikKf-%ZEsd^4koFPjq*M)?!UeN%b4Tjl-q&2&0qkV^4zN z_~exBixb^s`%qn~pVZi;GC{~@n>PMSD4%yo*9*Zy$4G9Bc3T>7lRlTkijAk4>QKA* z<$H;7BoqHS0@<5f!S9_gF6IS}BGMj$o9a?P^;o!;CoPrdVfj2=07))J{BCy|FXO@8 zj5r?>4lYFeYL}}kCbhDP8?;(++eHSuMRurM8pUP9E>m8wA@b(n`$`v0 zV4_Y3X^xikF@#&+tJ(8{FVjolmCP}AVn;s_EEmVMhsFGb%j!46Y;x^WrIxiMCx+Y> zH*={0KEx`SaTQHHU9`PAq-9+Q4aYUjT5u+vE(1;OI?* z#S07GO-6xFkaI+Gxnr!QQ|dEev!XqV1hr>*9n%jxtu(4w4Wm|CXB*D{H!-Wl_1G%6 z%+Qh>d)I%cly2%Bd?cw|&GaU>FrcoRHJh$+k<+M%J+0&5vsq1LU<`n5nGE<1 zU75(F7?sd!{es(uoYs&HV6vCv2(G1Az@Y+ZrESm(tKAm{`CK5x>E;+a4*u3S8nkK> zS^Qe@>#ZiFRokC zXdkxbaeql29JfOeLqStRHWrqd5eP)7Ub+wVgVn zbx%(l1*=dVDrBRi1|`=Qv9({0W5;5&C^$}|d#9%hzyHZ8^>!oB1hce;HBiq%Twy(z(!p=)^F{Oq0ce zq++9DU$0;gieJ*j4!h@%^a5~bx~7=BQxCU#?_{-u#HT!de^D@88-KNzH*Au(xGp1h z&)h|e)dK%cPio0ZX)IJsil`Ta7sSm_osEI9{hE^c#Yxq+O1MQ)YO(s>AcR(Y<{!ot zuozOkkHASUkoBGSZLjes&&v_n*1uU+!SYC{Ko|d7P&a64 zzL^jzjvB)R%I_JPLs}_*76*zA1NLW^x6_&2#pyE%GRq`~?BF+do<>91N`g;!kZZHbBr2M|caa(fl$7^}6p?3Tj!llpe3(6kZttmHw%y(; zy3}qRVX7VPwC)OXob8}h!Zief82t{OLAKjN(_P<}nPQ2DF~iXmv82s~COK9H(&1om z@#tgAu5X;=(0Q-iJaSVKNeK8%JaM}?#M?TZr+lVi(GBa`{xvFh8{?tm9rnnFw2Y0+ zFVdYN{Jgiet)IaQxx`zx5^H_}qL(X?Y|`}v2~bqrI5JuOh(l}C#-#;JtU1_uaJ~v3 zJv4goSvJO!45p1%X~dc@=Z!a2&*-79;5^YR@W$I`BTmePSI@>y**E6?6z;68X(3pX zhlzO`Q}27%S*cK}VDpOaytzy%WBo60j*GhzH9j5b?dUw8=)(Yu_cOGrZ*A}r*4Tivq3Z&p^8`@AX7wr5c8&?U>rv^lm8RX z^|MQGcq&YPG+vYDK3J_C;ldh4VYmPETkz{6Dp;zti%9Vo;+8~*w$PZotLxkNTKK1f z3p`Pzqi!#Feb%v*;0$t#?{e&JPz}$flcYlYt~8QUPzp5UEOIRs0(chmwqRsAF_fDP zS<^o=Dkp}YmJWoyR9%-ReVD-@4v1f&L>?AyVt9Pe~rimr;*g9EsPz((_NJ{-VpNI!qPS!d)FyAUOW zU=jeGew%9oX1}pyqmw6w0M2lZ-=CTQ8q>BrkdtdPGJi z9eaPv3>ix`*Dl?wpZ}r1b`w^dJK9RZ1L!UgL6x;%d=LQ0Jbx1BffndZB6nBM^>l*W z)Wp#L@S%~PYO3r0G0WojxU!Q$&gAnvokwifAR`?6t;NGg>$z4YRjA|XsUb~1d1Ek; zjMNTuz-Dd98a#&YiKac@~sg@F?}^VCGl)!(pI-q zusGtuSoCaBf*^dxGJF4-YYa#300VcQgFl^lA(oa3pXR}WC0eAmE+f%%z*dx~DJ=d{ zZozZ1+2|4smvW_eExA+{UGywmYjC`|tI6lfni<`TQAO>T9xARc2L#DdopxlTOM@dG z@CivEnOx3OVR>C&_DcxjoNRHsA4!- z%OX9R?^+uwk60z;?rn!8RzL?3RN2dd{{Cv5W-8@vdb^`)ObJX$;pVu@n}B*Epx~JK zAl31}TqdES(yAW`vAWuxiE}f3s;+s+ss?h`;D2nixMI4sB^oOe?4*~kvNB%DhT+rV znCe2iq4`ZA?28EAdWzC5i*wn7BLQa>dhOKP@44GkLdIg4Sd*J z%>g6eY2#$vch4}wFYIt7Kq{2}3SX9DNRR#we=wB*PB`L3O7e&$g+W;ch0E`|_ zTYCBq&9y69DW!Jf^y2{vJ+ynLi(Nz$PZ}Qc-jd&QO#VVl_2R5XZu-pcZqbeZFr#6M zQ6P&LUn@DJ#!#&BUK3OH4_)E*Zpn+T!fcuH^UM7Z+5&HA%NekDngjW+`3q+&CwGd| z3|DTo%0nFJn_nvhxk#xFBA!vR!V~6a4T@vF3PYJkal1Ju0H04o1B#McI!f z*$U(ti$>`qJEbJ=Xs}P*`ea&NQE+MkXK^w>bbBHGMuTlXiFu<{Yc-w#cKE&yn;k5h zCs&nDy(nc+YdM;vp!QsCF~ty);PQS_86&t3Y$>de3A3M{Nurt0JbiP0Gj@4M+R+yZ z6$~nlZ413kgC~F8M_xZyd5OvUlV2frXfC6_K>|tbNn9VH3bVSaxx8l7pCo& za(&>dRz)$SOxuab#K~+v70YLsxD=KwHoR8a6>ImAv#?TDO)^q&Q@YC*wxp@5vE&$f zl?R19{r?1{bVKgFk$L_F>e!AYDEk~dn&+a6!Q+@~Bu=Ra0nPQ=kWlPI0z@=+#N`ee ziqRJPfna?M+c{|ck}S_B%SaFSm1BMBGFJ!5{!8|^Dxc{D5K_%15Rh<)jWK$N)xcML zcQ?=ReV9*Io-cq)uuKV#Ppb1LhAx=>HJh~kFAUichJNtfP3`*>+ zLNsS>0)D{rE!R|}g3hbw{$}ID=$ibr_OHC02TW%x@m}gaE?H);P{G`B!5k^H%Zrgj znHDvLNo!;IUGfnc;|2VXh7>TfcN#sp!LcPG^X3)6kCH=#no`FsN2pYXJILoeB)?MU zz~7Srhk>BiK(*GSWAfk0%wqnP{5{0iH8|6ntyoG1z+SPcv@lz2uIft`8Kaqv=`}{B zLzS0=61~a8D%m=s7y!3o)imP}l8ABeK=pf)Czz8{@j&_)B7Th|AaK(Omh@R)2;4Mt zoDoeHY6+Jg>P!VxA1$2`Pkjl)bP;B^KYT^a{=Eb*H?PsOR+!{4Ih_SaF(l)8Y>A;{ zwU4$PO)=?qO%E{yPCyU;=k#Ehu$48I7x**%;~>CAmoVqk3>f|NZ|7uZqO)nz4fvUO z2S>DZ17D9bl412k4qnLjjzNb5)mN=z8!DH}2mvd*up`L2hti5pts~FQ4&arlAs0jH zX89PD7qjYGw{H8NyQo%eBtqFop-FVHTa_96`}N(Z$?{9Ur)2Rzde!2;BFir1r0rI- zF7ujosnvn6b`AoBnF#k^x36^0>=;U6b6$IroAasiZOb%Atn1IE3s|>|%>lPFB#q}G zdJSoD;;oF}kg%>V{CEn+u5wINx9n@~ZaVf*4H^12?26jFc8}7r07ZQ+bMuq?UWPyM zupODkAB1Xj#+xdGXxMV5(fn(-kk zE3pUK2Xzzj;yTV_nj5+O<-C$HO_wF|g6)>t@%1p*+xJuuwL|ODTJkUm zm#oJ|lUUU6aAL)vYGq3$L0PSQBE0?m==_Pnt`~CQA}kEsCcy>dm&!5Dw8wTTSujz1 zFtdMAETNGzb}%@MU}uLQ;(nnIQ${49b+jVV5{o5fSM3(o7F@fki7+inCV_lps+NfXbr;3gcm7n>w@G*^u0z&XuK}{LFa1y>&9wWv$uC&%D6A&WVOvz2Ag9n5?$-I^e5e6` zN`b~fxnsH}ZvG&Zf3l`4{mN;p@wnk^~)hGJJcZEW55YheRD_HbyS?Tmv`ZEJ^K z`qLZ2XM~Z4idMeS8x=NdEHbp!VR2Giyjd0|oDnKLpxHD~iwcZubZ9Q^ z&HCfmd0s_Y>^G9g^A(JmU;Ms@Y``8caOa*2^aHJH7o28~)-kEVG8+_>p0qmxy-I-E z+gWLbfTdEGvs=pZhOdT(u@-QC(2UMH3S7W8ZqqsY#KwkS1s3sJMLsOtEp<3ml=~QA z!y`ao8kc>t9fti#wx`mpayII1NS^ae#2;0Wo+Fz`fJNzBxTrJ6%FpQkZU5p>T4*kS!CvC_ba*F`IpMJ5<<~2F zO*^M^L&?68ryy}K=sB4@reclAV9$2rLPPlv3*W4G%%YRal4j^5lsR)dEBuv)*t?ncCiUZh*aUA+c=4TPmEfT0*sUs^x zUd&Np#QB0Ppr))#WsH*`doT|36KDqI*+$iZeNl{jB^C%dD?bB-orwonu* z!_^AicNbb&aaYDs{3nzWI=oUnS3g!zQ;F_l^u zxAe67WV&^IWD{~Y{Flo!Oip&ys5jZ6a^0H!2)RLmx_i@5pHLIl;Na*xqYWZMs3`=>Y zwvJEA7Erh1s=H3wS|byUaJ$j2)!Y`v_nQXOJ?d|czr0eH>cS5XuX%o6eu-ewHAA!i z-pq$0Fu5jm0z<>lBi)wNHNpnpFus=VkWh%VKV}p!EQT_Q+wD<|5bJ(IogF7mgLTVu zT5fXNjfr4RJc)Hjeo%Z#(J*k^EtS{Ut_I}b>YX!9&9tyL#14xNFkPyQK?ri2Vqvpc zy$%+(ZlpoleCj>F01zukmUc)!Uv6tSJ2NG6B|CuAKm8LGwSz z2csjUddFRVuI43(eLt-VJN`@nsQ-n~w4YIc%M(9{sdIeRHI-E9TwgC@s2XoDtJz4x zd7)lIvz`jpRFh>3I~??*@8#SIdx5d|I)kd)`};+-?Q6BYs88-Z>S`0qn>i0jeorz# zH39KbZ|p-Eg*&bCb;6rvt31VU$g-rk7;UcN3>9-rY-p$vfRCLlsT3dIpQx7<-%*Lo z0U?o2m4>H};?hH0jPHQ=;%5S^>Yw0@LmgP5%h`-=tq`Lnczav!=q5{LagmpGoQ`HW zITY5>JG@Et*ts6Th+vfwM8vAdGO27$EZ|1p74YeOG3=KmU+<^>Y*!kE)T0#vX=;L` zy|1HT2C)?OVF8|ty_40_{ecpr(#Th});P)qtW#{WcF4s zLzk%ZFVo#mW7Q6X7Y&M7Mti0-`0Vz!Qg_c95q>{ZKW9SA7Tay4BE*a$nTb;AaVY`ju2jj77p$RO1*RTy;7+)}wq1d!d^iz@Ck2yaUQXHMyyMPPpfMMV`pdLDB!n z_e##32~1P)n3Q?gI!KcL167~36-0Kbt5>pA;)o3$CWF7e710POLoOevl4Qe~aF6z%d7r0jTdSkOeI_m`4U)T4RIrb6c^pWnxhH`Y&Y22q5y=ufw zoz?=y89c?7FqS)t%4bcZgdhnmSEyp7}f_hthw%5%=~Tf1zmC?0-VM^g-)Dn0u|0(AEPC{3tfRssX)!QF|-kAld)yIm@Ia9Z7^Q^u7*f(d=jdCgGT^;<+vcyC{<|algk7 z%5*68@sLKR>joTMNcV_-uQ~m?bCCZ1HsIt;)}6^C?2^a&`Nni0Q$RH6(TD}U2^f!V zw&kW~Lqmj392Y4CxKN2)7=a1Jm|4nC37Cua*reSw%-mVYNtbn16&a1#f70{5{*nZw zOyd1aOzl&bCZ^L!otU{@MRM92IgsiyZd0JLR3)0d=gWvp^84QY&H2gCC&dq|?9Y>* zkG=$8ECZ(UEoQ#Dee-X6@dt*mhh{(T6H$^EKHYP~!29Wa-saLH$wbuw^3{pD@8K0h z>lqRlsytZv&u`3~#1m1u^(wn92JxN7hC4yn=+aKNbRk=Ul`M7&xk!AC_DU5VO#Z^r$<{fYld#Ycjt7QM)l`U_F7@E0P6$gQu_?|DUF zhczAsaJ`^^E9(rP%l;FR+7&ToAR{jV z_cAmfONcKmmuoUzm^uvQ{^C3(@U^B7ZL_q>w>Fr^&B8ODXo|e^of9g(=mv?6Mq^oq z8Z;|0__aSgmzEQH)_Gjr_2PXwX?p~BRGyb0#v?SC&c+5!rAPZeZXCsXqpwuZ>fX;F87RAZ_-iu;+g|BBy7rWbJ8xFmk`=8_MD8~o%HBRG$C8@GJ5k;KUkMm5)F2@ zh9{e$BoiUVyNhvC-~Yt;f~B-IgvNW;&s%`>34kBH@Z4_6@bMQyWCgUdKegT0{uSbn zD7HPJamT~E=>40hOxn-W{E>jbmAW$tUPMz*anwZ8-MM*KzB0=2(ec&RZ%k>UsDp^2 zIlURsR{jGr(o|ByKk;xT1bhq^tQ?J->^W-X{z4eL2Qb}jk9etpPkv90voX%btl=>W z%)=bdYnVsO05-LT@#;pbtg}Xmqt(pul9UUViFqs#> zn0QjMKs^YRH3$YcISZNGRMOCkONkHur~K=GH691Q&~66t*l$?o2={lvazq~H;ksE0 z!c$00a+56>){wH_v?vI+mqs3>l7}xadS))|%xvmCPU{1Ky=N<#gtz{WP~-az25pt? z$%;2N5xteWZ(dQ9t96_qS)%)+xD}qltG{i5lM%gTUUL!Na(1PU{Dlv_+v^W<;RZ?PlsfYj7Pl0{m|@p@VAFe` zh{HbnP!@pqA`?DYEY-so>#FL`Wgm(k^&1qn7)KDDkQ*Z|(# zy-i6f6*#fCurB=o>LS@j+Gq)1XQDG$+!HrJKgI_4bZhKzrGs|Gw4G<2f9knLq+q%p zTFElpuArnkc8#TokNk%F174fNe<2osk7h6DgIABzUkJV$x9Q$*5xoC4-~I;_=pU1B z@CD{euj&p-@9q1~<>`~RuiTG)edC{wvJ6+e+m*XWx9CX@&$mYgZyMj)NF!UORFQfG z#J`wv7<@wjYu_tl0{)QT?}vO&36&hC@Md33G0fbZL1bgNto#!NR&oA>yDaN8#BC$H z(&{UOuK6xSV<)kniGNW`*J50Fb(>gbDTa)3av3aaxyuYnFdNWb6d!ZkDD*g6#~qwS zEfJObko~js6!@wvAyO}}pXVRxQWJ=@S7Q~vXNT#6J%aZKKh~TDxkGF>b^luD*J?W; zy32MhSX*+5h`HtWW~&+K^OiwXMYDbGaP zdveK7gr1x@3wCKU`|TQTLZn`aM=ng`ENU0yOmY7ErH9D-A>;*yz;9@-rG#jxHdq@t z#M+L~0+on(R0a`NjQ1kptU-6XRFC zv$nFwi z;9gE(YfWD%6!wSbJ5)j2qkea8MtD~mw{_Te~MZ@!?*W!aoY<|T_tRA%tDsIsX z!&wC5usn~pO9zJzNyq*wp4e>DizoimuC&;Ty7ZC0&n&B9<6nXOyU?w>Gwoq*#roMXye`-lz6tX`1xk zo+i}V8wogF?${!5l>{syi*t*={|oVQ8~ztU=H$*WAXtFIYhjU=4|AOVdw||F{>!3? zedREF@!aYGvFyv+eeP$9Vz;Gv@_;Wil2vCe1Y5jjYuU@hefn>1H-f_)PiNuZ6Eto52)y(JvoBy z5;p!d(|z-(CLF(IYvQMq)-!$B>+Dms0F-Z_GsG^{jEQc>ivP-*}IP6x1QQ%_2(D z8|%Nb!g>1kl?S1(4-}a3EaM;yToZ0Uo>)2j)jE!(M$p~KpSU`a7OfH-^+`*9ijP8E zkEGkUYE!pikad;pqq=NvKkc!s<+gr4JUV2*4tXx_%4qPyfRkF*g$!QbfFErbssw6; znR?v0(ugM&Z=AL4lODf;PP;};CxrM9CGNaDs?Da`sdbqXnlHFi|9{DMfbl|YFKw}m zQp6)j^D7c$;3nd(j`yWD4e~K$x*^U&F$9v5QBvVhRnzqA4g|wX*5?d^&5YOd>M)e- z_SO-4L4#0KaznZ4jm9PE!C%wfvDBOsair!QtFqyBtK%V=dAzyMnOX3pRh6639rT~X ziOHaCCt4H}8MA>8%`8S6Rrv59?L-f+Jcm1jij&Xh>7Cc@>$RaEpzv5fKQ5z!l};TZ z=GY4jDREk|o_LL?A$>vyESVouhkh=e?PBpKd+QTDrwVQvS=B?mKTAtm&8siVhPnQA zhSzT@j|hD|@!!1iQRBM35VfWp>ipK2hz0s@6!Qs#_jM%QSfoqv{z6dUv!V&;>;fOY zPkcra5n;*ttk-N|k!|UT?Q8oOBekJ{rB4XTl4nDB5WQxiK2BQ}9nXu6B!`W!5t5Hl?WPGFHpar-SuS@_&Sh+JC3<>bx#w zJbf@WJ+tcFjs}cyZgRtqUFMp!V+}FM7OO=-hApzp+u(+NVlfk#esxnafMVP2`~f$| ztJL!i_hhMb$pd}y$=Y%f31O@*c2DWll_|7IrL6ksmpP{U`-BqVw&Ys-m+QjS^{Q5b zi%PExsdy(T;y&-SB8FNM!{ZKz*A3Hcjx5w12}{mj9Lu{2D@-2w$BhRvKN6>kFna=~ z7G4TkhozcryB$vzgGt9~`EGfg+e)Bb>)^d`BLsAdr=GzUBov?bY}enrG=MjcX+mG` zvp(1IUVZ|l%IAHOR*>MQ(zt6F$%)lvxR%6BlYT zI*C}eh3V%aU6DDy#*};b{iJsEx&4`8*kopMCAfLo1WTPsv8(Aa zb~IE7Y;LFCe^g`qN~|33xj*>?H9VXM%$PE%#}b7uQ)s6-ioJ_Bf-zI$#|rmgiw1LY z0g_5QSpSE*w~A`>ZP$Hip*R$GcXy|_dvSuhYbb6ll;ZAQv=BT%aM$2&L5e%Y3x)Du z`R4l8+;i?d57y}(BN@p^a`fhXhqB9{hTV-FYiVN##MpV{d_EKHNi5`=>+sBhvgUF2=MjxvEGy% zw~csw{r$>OwBch)Ex7~kZB3`xvuWqf zW3emo@$X#2N05`|$O$vxXfIDNX>7N>&TVxqF(`5GiOk(d)5cQ;n|JW)qw>sNJsHFg zHv=aiae_pL<9RI$ha&oG(sw2qFKOBEi4R_*h?I#BzvZ0ylw8T{B((}|4BJJL#B9~+ z6W_i~&xn5;v-iP`M~se@oPbPh-*P|AHRG}fC&=4nj8s5Ar)0VipYCG55gOiF?{UAt zZSw+)fzY!n-B5|72qurHbmfpIC$`M{R<2FpQ6V zr{OFRa%Eo&QU3S;RRl;Nt>i&i^|VKS~-mmN&7dcr1%Ah( zeG`w)6kOx5I=aQrq^IlGiu$QZZ4BbxZ?aXcBvoLywz7V({eBq35`K_@rP$^8z|434 zOFJ8%7Ji87njnHt56wP7Q~Bkn2dk`;%$7_THYN<;h80aJ<0^_C=i6-CBiU1Rjk85o06*45!BCr_1i#6(>aseRFtthy-=5zhK?)i@acaKNx(tzM~YjT{IDT~(WUhW3@W7KJ{#*uIj z{$pl$*P_PR0XNAkZyw*7sm{0jklZfP^GO{FC#BeeXb)b?18DWRYOCfPG>Bx_s(j$T zT~k|ZUE5T1ujRa3&Rp(ALB!-3?96j3Z3`Ju_6LvEE6j#Bu&`_tHZ(V9k z+|ne=IHAV429t-#uJ}#- z0sHr!@@L|v=1rq<-EP?#e>L)-PPdq?$xKhT`1>D(DK*8AhjX{MBwziA-@6tXXuuw( zMwG(WFE*E#{C=JG=_+~5VwwG`%>2Ja(Ehsu53}6m9>TFU(p`~@iT7Royo;Fyr+{(i z)Y^PYTo+Y5Bb1r2ThOp2QhnCbP~+^GsPbv9mr6=Rj`&LvU@BK1Hz+9FEcXimiUb-L zO!$oz$aKhMOIWj{?r-)@F^lO{%+=)Elp@b~DJ~b8^lY9!A0le(+#r6H+xgC|QK)Pp zdRE^7?o+zn8PV4W<2=NTgJXB1$%R>?tYdtxS zJWl;=gP^5#mxhKALuTv#*lpF$SWJ;(7Tr#F|Fk2aDjZ=emS-R8x`gJ?yC7s+Ge(oN z?xV)eFKsgvljPV>`419k_A(TWF@i85G0$#p>uXDg>+SYRopBR`I}bwUK=(7BJ5UB- zcxz_C!bUZSOt)|?EU#hm1m2=i?#P9nwObEiv37<9uai(_LvBcRwuyjVcz{z&Ts7xi}+s7K>Ts>!Q zby|S43313vU+3}_bPGwBo0(SBl?SHVH24QWt$9PR#U~^o zz2re>`&-|p@l*KjOKzDr?t?CcUG!m+DIIU!pp2rZ2}vq&$HF@HwldGTgB?>fqoaz= zh_M2hUdymL6AR~lfc2e|_{J|av4@;2jNPA8s@7Oi!#ZHh##b_pzXY5-BDe#4gwULT zTP+t?XM-obe3rLDTd_*K$C2|mT)RgeDU;=dr$-(S`qTs0Auq6>f6mxrrW&jP&kUWz zKxNUmeO|-a5}?x1mol3BDotA9)hD;@)5kUez=ROfu6MoQL60;nK~kl#z^; zF?eS0kYfIg6S+9bhKM*Nhl7#(s>u9E_~|kK0uA=uG|h@8`9h^v)q8wh#FO#Tpm8fS zDywtXz&dW=h3#q*(^3QjYshPVr;tXn;}QnGOb2UOL8gvpu!AtYBfVLdizJUj-VG^c zSfR2yO@JmdLn>Z+WZJ0wUQW#zTt$-H#*l`GXp4Z-Wo_zHtI`6lgE5>g&lE7!INA0T z1ZfPnzqLw?zyI4h&*uq!!*VBr_~Ft}v{0?-kgLdQ`8^rM=Sf;RnZeou z#fW;YRB=vP_ucE_c>wOf`iDTk9=fT7@J^K_~Fle8Y$>a%@oEWODRy9PeY_`*0x`8wHs5g0U6d)>v*uS zmdjQ{;N{W1cWcMWm#^{3b{h#O@ztnFL|ls)PG~HeYG3!6aUTx0pK)vAC`j>xS+EXF zd2otHZBjV&q?afO97$F}|5aNF+x@$?61^IRB_wCL6OEryjB-yg8EL$>jCprAvzNF}wA0Ytrr5wizvw@=0PiiUpI3ip zy}d>}Gx7B~a)$6bL#Qo)ioJV;f_H~K;Wl}6c)?aunKKM))bICQ3P!%`Tn&_I?);>` za$w2H>T>Mkk5G(6P85;6|Jowb=ZFP2j`3&TiognzW84)XXP8pl>N{=9zTgq>(MeP3 z&gcOKaDF|T14XdxN7rPv>`b9*k#q`?@9L_OpG4jzCe=|^3-j@-NmQ5#VMyro6fL!K zy->fuhyH8y+(=r4IJ-{|TzulrR@Uf zGz2w~`5#^agXeI-hi(?%q-SOO%hWyx{$a>3{4OOn(Ny0u(L)_e)vx>e`WK=xcI@q| z={i+KGn{4Tl!6(VQNe>Ir((L;;pWOq-$9zr>?u8@4q8HfNaha? zo3bLQKlpYv1GxeXm(JTO%*;-)Ar4>Y5$q5(vK%vgTBmPcK~g z8Wim622wZQmhh+`HaKhWJRnT~3`taH6*D@*6{v zhxb8#h!?xz_`NqIkNRP%;wSyn3Ii+kWB`ym3iuImK0Ng^uq%k4WexvxbWDmU(@0CV z;uJMcXe?R^fkqmioS9JI5@Ag;H|O8(HJ2dSkS|L1PWxeaRxjEKF@43_HZvu=!-QjA z-%!dA3W7dY-cO`+{bhQnlm^$E(G2JDPGzXX>ESdL2GXBuHMVrBE>0#0F2E6QkzR-B z(Cdwv=C$?3=x zvA^%&!tH=8XI2)p)UqGndAlUGgNJ^7xc>*CET@x&6rzv{e5iDJRV|`!jKj^$&CzJI zHCAF^ml-5)rWi?@Nbx;eA`x_->V-V(e$r8eaa4#)i)CFfn0>ko^?cI)q99mOKaOKR{BDD}ar%puQ(>#)NP)|t z;%}yQM3ggZjG+Y&PxDG;<|%W@k(A1sh>dV|LF?+BW;V_x(hu5sCL65}Ep_f0E6Wpd zb^iZ(fbn1V^W>DRzvjCg9Nq0GbLx*w8t1s|avn4CIa%^Lrt1dcwk zPh8DlCx44VNp0|bP~2X6CnF-ZE!iF$o`$N#9qIt%et9#$-H-h7uh`8ZUl=auoN#?QpN5+CFCe(1b)k+qqBM3&7Mj~Mart(9v` z9veBRT)kz&wYw1N4IaaRREs?L$dowEXLL4nsUhPT*1I2PD0?61FeAO~bB<~D3beJL zjpSb;lWi0xjV=$sGmdo(zO)_sP2X2>jghd>?3pwZuQIct;_;b+oP1yPc#y0D6 z-f1Z7U}j_gE*;j5`h?)Id%2R}D@&lxl zB*FP-x5bM+;Dg`hLjv1UX%_2ysqDgD?GE&sJfg7LBToMN)~p|P%MiCS4>G}r`)X@* z@JR>N49rj!*0TbAN*gKIcmFe;E9x3UY=m}vZz`-$u-(*Z#M6XP(A2HQ(X!xT!5cVz zgsLxJc~M$%-wVsDb}OdEk>Y=yRGP9*0!uP1wtsk2_E|S3Kaf4?hQUD9o-b{vPCx)D z`I|{9FRB>~oh$9F7DATm+!Pelgn|YG`pmV5r15TV5N~jmUiz z?A(zND1ax-Tc6O)d)Y0fY;$R9NgE?eL$$2@WEoZ+ZtPSEjwy-t1A$ssb5)q_Rw_%f zHq~4;#5v2|d!8?AIl&W7CUqUn<*dfb>4#_QrkjHrVvC1~vw-S~j=6C1cGn&8IkV2M zUatC#$!3JC&M_RZnH#w=Rc@_6LRw8iSbf+vQPi$SDgK>Tfj$^O=9iT%mh}GfcjN1X zV!oQ*0@rlte5M_;ZG!Y3&m;frUR*C*nyS;a*=ovifi160@r8URE~xAG zrI}fxY55N<+WQbRn2%*0|@0 zPaBDl=Gxf2Tp;8SKPVUa^P*?{n=P5B^#y0#l)irT5TWai7<w??zav(v(oM8spwTqAmNpgx^_+q80oy zbuE-Sr&oaSIv$+GULr%d6)$k~$CqR(Y@)<{TU+SuPJ?s}qPb263P)GzHVF-MAQc6_ zbw;UNFbjXIWA~(LmsX>Q&vv;(tDj-jF3j9Y=S)*|e@p%J!O%=V732KuI5ORIZgT~7&Pyz@5tYVJg3uI%-;40VC9Cn{ponxT*v;i%2&D4}4KzUEnpQ`MG&G zeS)ugN`!gbxpX&G%_UYM@sgjmR2V7B3~A+1-X^pe-x+!eQ zcO4o&kK0}+d4xo-!iTS4ee;^aDUq0r!EcVKoncb}-we>2ou2DSM<&-=5UXC6;xnP@ zb4^E<9y2p~K#4ChJ9|I%IT6`%P9mYA^!!E|ANmy1n77b6Afbx6LnDwi zmOfpdjjFCj7$lGtTgSD3I`H98==|i`t)I~|v;aQ4ENAbv4tr%R`g_pZt%>rym-sKP z&X@ecr-0ADBn4r?XK^iA@6Q&!ad*37SVm-YCg#w-E`KsgM}TqdD*)XyN`{ENSsPmFD{k$+|Xo@XtNnP>v1Vuvfs>Yb04K^!zAR zDKRmwx3VJnMNDcDo%R%sLer{fo}wK|QO{x3xb8a)^IqB)*S1%_zb@Hju_zjS?B|Y# zxm~Z{nTUW#Nj3F$FfJ8bICNmd?jW^NAGdN)3CMZPq^1W}ee1~2{8hL}gRH^v;I{gq z)MBx*Nq`X84Q(BtQ{p0MA#m+htC8tot7}<=K+0lO*+bmG6LsGmq5F3G)CGTjO!^xF zTEa+d2HS<);7nYfZDlT_ai*v1~I$oC?A;+(I!Fqq%W-1w5T;RbUPXUN)KqYb6(*_ML& zgvY@IlL4#i=uDmTblTEZ7&t+)u0D)QI zj!PkrZ_y0a;-)%rnrB}L2%IMF$t(*uFfD0?*|l`!Xlf91};x{Vj`!!qF(eL>rUwnOJ_jP6AY>*sMMHaO*J+ ze5&R1KdSxf1zaorfi>zL4}iX#bq7^8xYiL&#p`%p>z#MNnX>mV11C`TGIP)mP8iHb zA0WYps$>+6ftAXsL@z9__UwDDNddVn9o699XZs9Tq|yGz0Lp+T_cBdL;8 z{Jgc{>_cT_^|tKdzKp|vHoU5%yVEqRz4Yj-9CP&P)~^qM()66}8Ir?g_Y4_5n0R{) zQe>i=YxlA?3H7QmCL;gBPz&2Crr ze(}SHv=6E4j!TWad@@XUl|wrhUqs4(L$OsiMdfK&ZUr(!jjBg#jR6Yo~ufGhARKnn%lbEXe|y?Kmy^kVs`b?A39d#d!#9`j4or9}SCS-c;O7fDzOPn|@p z+O=(THBw4*_iUAwX%d;iM+Sg+LF7{j6<6b)`?q#RB9>5rW0~F1#(ke`>TzI+VM#5e$vxOU_Ky;cy|vQCaL7 zvi{&lucGwKloECZSLE)YQ2e?qk`$bVyqX4fKG{cI32nk>Zi;tn%Y7$QZBE}zb06L- zH-@+f@vZ7bQI37%Pr;oS#bVm|`N8R3bo-~N0QKYHT{A+=xrhqY&#sv1aBkpvD4V@1 zRhHC;&Ox{d*WNa)PlOtNx%0W<0Sjj@dz(5TPRq>+_Y_JvDRl^3S))|( z@H4s|d*vORRk7Lu=hLJId=>IbDE|iSr9VqsZL&UE0j1ej>Mgo!U7qYzM3i2N^F2t# zWhafkZUxN!)>#vBVj1%;!0L;t(3SgYo9gY1KnInrm=Nt=!$WT-0VLlMe1ZplY6~Th z7pwj1fH1nT8v~oDuzQ!Q8@YS1+rv=!eslZv?Od3i8Ok+RXwl}LLh7t*n7J}gWF-St z-ed0YYYosG)^iYFeqsJf(P9@=Z{3+r%Ca@;%n)g&?O`al{;E-a<{|&vT7^8@&AfiM zYm+S+75G-1E(h66j$T&W$L1i?Sl+w;GQXi^vsKi6?KBb39z0#|CG1xVlrpUcP{?eUT~O9jj?3@2R+CTh_e3oH+HMMNDG!4x zl9n6CuEKcVy_NYtRey>@FAfGVv>>Nwh@0~k%gCYr0YL+gA2dT6!9d~Vm7w)E>!!OF zaV^W*V)1sU$?5y7Hj`827^vwJdywTbRq_Z3FifEg z|2O{pX5whWQCi$5i6e_^W%GA1;V@`gjRy8Az@W)C2HOOhLx&Wh=Wu4(X6o)u7^!Li zSodgrT6^QPp-uh+cRvpTAE~NHFr@+<J~!4GiIT=G8%Vb%v+Y26;o(yMRIJknYbf6~qUb z{@9AIey8tiHuIrBZiNCCDA-fM@zqQ047udU_xQ+*)@wou`FLI0xEGt9EgdQq)x#g# zVY#sLjT&%9?R_e?2<}pSxAJ`>sGY~acBNZnP^6Uw(eZCI0Pdh1)`)S}CKC?3bRI@c z)<}!OI??0uUt`!@O*kz}eJ}z;w)-T`E*;d#Uis1X>K42lYh%-iLsebky)Xkw_`*_4 zq>0?6Nhy+_a&N|*8bo5^}%8Y+5SD zrl=izuF{+Yy?$R#9B*P)!URij-;XJ|Uevh5GJ1sww2`!|zN@T%5c(df{(d<_dA*$S zWP0Wa@VY0Dz6stYr8AMTGnSd@N~}(9Boc{%UgsCj{#j33xoT%s&!F<6duP+@;kbDE zHkHU?sh$QGN;6G@A28cNyPvZhX4!r&1li3Y(jNs#8Vt|R+UHDDQbZsZM0y7-Y^8D#{E_JZznt9#7$f1~ zwxUgKXY}JRv&XQ2U(FZ7NVR?88kd}U*d>FnNt*rhJBt3A$}20;@t2QatAsqWi(j_0 z!tyG@W=Qba#9ZKYh;27>RH3<7;C|K&bS4Cht=(wVm!}XmgEyw$&Gl4)Vxg>RdMbM9 zkTU|!XSbe-i`rJ5ES=_`!T&Q{=l|5EGZ{$4pE@{=ORBTD^fiz#W;O4#%zWxgoakaz zV8B=qD9gm!Y0YhAJ7f_8bMbM~M5?-TiDDF;X`Sx0N`B5U`be~3{W9|K`_wcpP@&~` zDd)z`4P@fBeYRA*3ja?QYFS;vW5FIKwIy4*ZZhNM{>67woJ%D6QDG07T(W|>H!mti z<`m)7lm?MBBxR+=2O26dLf(Egd}&&yi6omy1df|OqZcq>>wBwXgF&4#M?Z}h}`F*ml^kO z_-Kl+PYAwh839YF9D50cXrDgqdW>a`c}l7M8W&6b2O&H!?vdj2YtFy!Z0`jj$`Ie2 zACy@?*SG?gC%=||ra>A*A5X!IjUIEC9U5yI67M zWq>wt)G3Rn&K^H_#VGxR*eak@RHuIaIrDXj(M=7ih$Z8V@x?3XE%w*ez(-?wJPM^H z-k&)wz&oh`mx4ih(aC$4Rm`8a^qA%M0i{Y;i!&eAcNlR<3Q?>-1qcWJW3~^p*Mo4WyjVxBw zG2Ui#DIH-}sz_5C%@E6Cr%~W#KJm zzjFLaFltt@#{Fb{?`ReDsB9r&Qd$ znV>=a5c^1xaIV?b`#*20Kja-YUYDHx9wR4x^?X(s5A_tzT$%jw)!QC-_({v8?CRtj zTJ~T>@9+;x*6+&8bTkg~TO0 z?3rBC9y>gcXGn=qwEmU+#mxVC-`Z|E>rcb<6O%>O%~p~nv`)2%sOxUrSQ_J@%vNz( zfxwiAE*l(A$|7@6X;gjWE7<5TT?ZE+t@!T@oIZ#xkMSin(^JAp#6R_a*2>PLC{$JN|df+pen5_KK94Q ze6W>?_EtN)<>q0F*8<Ne+mY!Wkis z8PAKQ=_Ih#&Y0O>y&nEds)3;#t=ETBwrIJ7v5jtdctG|8_V?2h%KgF`#D@7Zk9sMc zEbxYgLWOR&W``INi9U@4L+0#+-PE2ARm^#TqF|5Pk_QNU{a_|3Z<=Nxv`{alBzzZT()A3d--Q8(tjJ`Kfw*FOb8_3L& znBp2p$w6)BQOc*PRHkl(|B+R11<#6rfJ%lu*~{m_*MkE6&GU%V<6Kx@-s=iQ!#@bB zl=!=vd8JQApKJPQRw?GPmq>(6(u#(DjaZV)(ofXfh~m9TfK4&G8C5RE4Ip7CTR`LR z*mee3AkAvXTV>6c*L}sI(~m&sp*Tr9G$NMsKE+m>6M_sCB$s#l6p9#Z(8fHQ1TsP& zvYt>aTi&V|9HxEBE_0g3#}c5BHZj78av8pyYta+~CGIkJtO3}$V$XUiE9ci9Wv5>_ zS`+l~8NZ*mEm*BlcHK7w5Gm znMukK68fEUGdcirQ>Mu@XBu^Hl!OR}cFZ0|Tov#QRNj{-Vu?T{cOKsc-gwcgh1~17 z7Bh3!n8}}az!n?XwtX@;-BxtcYuRU}XtNvf%jaBStb0+|rfj)buws$;nvS#D5PTH^ z9rf~YHW0|nE#`+^t3}9f9hj3zwW>Ft7Vl(LO5#H}3SL&u81JrWj`XI~TH~ z!mN6R4t2ZFoh~jxD%szcg~Wr)K_Vc|G}&Per8R(YE>GyA?xDiFZeM}y6}O^9y8iLW zZJyvM7YczuC@g5fa|vdMR?+Fe!je12Q4m`m(h751#Htr}I&(pL$orv_;5Oy`whfB# zWAz~3C3C3Sxn<}{mFcZI4#P$pik)m^_VDZ*j`X*g^3!-^t7V=if=HAxs_UZ_yr-Xn zCMQK7jv9-0hlySTx_;N4K6qQ2KQ`UB3VUT9wyHhoz19uzPiD}K3-hO!FyAkTOwEjz zd%)JyqDrQrJGj>N>pud8h<~#!%S+gLw@y(yRQIiUmtB_fc<8KOutn~kxBa$adP>zy z|H0-!)+8%Rm0d*y{3OpKAc@MSFYq1@4TnHE0;MoUJ$ntxKDa+lq3+X1ckN_Ko9Shg zH9bNI^?$Kq!l+!jy1{4JW1ie7^>Nkjge0mk>~x%1sIHf=Gh1hxslt4kYb7X3>SN45 zO``equa^OH@IHR$K%?~F8hmSt+YO^|p-r`hcSglz86rTOmLLKNQdFk}0jE>q45y$! zf-MW9GEYuh+8xP+;PK;A{c)v4+LB91tCMx1?-f`OPdX+6> z^M9D(=<$0qzX~QH>p?zK^?FVXn&pH(h~kZ1%=O5Za1 zBOb@QSOi?oV82kq9kU}yXTmo$Ua(d4>`lAn(}Tu(-(%%N@dH7PIO5mm{!@TgNeq&_|j!0 z^Fs%z{uG5$f{{g>gjouh?~@hFK3j;8IN~;HrUeRAg0ToxCWI#o=u$5>=*nB+9qQ$b z!!xlWW+|bi0_U9ZaZyp$EK5vb=G5FE6E1(trxzazHrvw9G|12;P&%%aYo#_5V5e{V z2L_kJvr3l{ciH@dkU;!`)_dVKA%5Je76>g5@tg_V>1uA`kZx*#?&dLyFx|z=PwQvd zp?dl=tUmI}#-}uq#nu>ncq>Je_=f9AI=sKZ(`chmbN?=XW8k{)d;1LX=R~Y*bHTML z8Q$S>93zC%+~9dxS4(}$YX8Q=_YG@fgZPo8jqS+$7~lk@Ts+_0dh!Uo*kvAB%MROX z&``I!1DgF<(JB;q%APpIFF8M0XVNW{D6Y>Y>bxBMr!5Ww&EZuESb(%zFICYg8YQCl z6B`I2x1*?}JK4L75cFT*59=BW!GL`BQ}TYtzQw)MTICX{{lOyPq&-#_*=OB*4`Ez~BE2xu_W0Xyfh zSsGD(nTaHr%4Gr#(4e!ygHiVL6y9mq8_~hJ#6q#jdRD7X6>NdpzEku1cjmcOTh&UF z`Kz68Z8pToRH_>EkdESQW$G( z-r6Eg>TFIDM8i+`St#@_KlkI!Z>B?hQSyn2NxLNSIepTo;=M#l30DMt4kE(Z+zdp}1O0%Cz1`jPjS~^;hw=KbvAJV11w@u!V(lP7ldCByZBTV#yP@k#Y%o9+0 zJa;S0&o3T^CF)lJ=d}-px(iwwfzOM_{1!o`nY$ck3~QY&0@X1aFj3DM%z^G-rM zIb$?OvFC?4UkMngXsX{qhnPkAxQVkA$L7SAnER8yFkA&7bEWCOb2V6U1CO!@1K7!* z;0bMsze4YuP(^Z;35F~M-%@8F?af#ij5V1iaZWzH($nND8vC@M;GtY+>)BG)Z)*Dh zuqnNL30-V%%-g8$qQ2P^Kiq|Mw�-CD)4}CEuD9aLx4o7W7Op81#B=tfzuZP8tQd z+`AzbChj1%`3&%xjYDXrj}rgAFA4~v<>0v?m}EnjXC2V;UL<*Hm-MaJetU8b$x$na z$8~tH3eHkJ;MGsTqGXwc((*oBBP}2uWAmI;1*#f>;1s>8ZReQEf~7|JU*P7BiSwTI z`7;m5sm_@@q#5pNU{{@QKAe@n+<%G83H&yfpOC?X;;=B-R+eT1T;pFtqvFc<4dabR zZp$I;=AwpM_;~tXA%_QMA{ynf9RNS#%9b6&F#l2bwu$=@KWu$3Q{p6|^hJDOnRUX; zGC)zT5C{pRVQRNwOHX1?93n`NucDJh!;81^1)t<850cyhOd|(xs#T_`lgWf+_QKte zgx^ph3cvgT&`0JfX?3`kkF~iHRDU~NJ1#%rh$CE7QFAdiC>)=-fSiR0)Jh2a=!fq^ zgRfSW=1J3NQ--_0zwI-6j8B*&sURt|YI{{n+q3dGSK0M4Z2AG?O+tf`Cg|JgqCa+N z#B?B5E*}hvfUSEAoqHVLC2G+<>PX+YKdfFA=;a!>)K@isY` z%;t=85TwRpZZ33^D!F-3Q~jLC#Ez3>=m&ovv+qm;x=T;4y|u^y)^y!H)6Ho1q)sy7 zS!3Xf1g|kVH~pwDsP}R2XsLBmLBcO_%W2U-cAb+cK`qJe@V`3vkP!d;W!~45G(Vha zhW?;Y-jzqPj3Q{`~NMPifiDGWBNd z={KaGU9`%c3?@u^7!zIz;Ht`+8-~aM~#D97J|GzKal0Wj%Iv;PGHNP`(bFSLh zA^HQbv0IYOg6XFz4CGm!2oQEl**UwIFHOZ*n0T~zEW`9J3%7bcIR2SFyEJ8Hdlx+= zL@yBTJ6RUQa4uuXmuD4e@_jk0BYc3~tF4BDEfdGhi`H%uDRok=5}dF$5n_C{(9>_~ zDK(X8Q~+7CgFiB1c9{S-PRImjIjb84rmB+(b7NICfj@Q9%UQLQq*mZ{dP3pKpPhm3 zGN;K2V{g;u5h3TkYSU7iQ|?i77_bX_)SYq=)U=j@nUJ z)i#l(!_#tmiE4=&dUI?WKFG%iUunm5Zw-K3B!GxW%v6?Fens3ThITV`d34-RW zMvHa_Ct#2cID5ytTAWz^P$zQko&Jp~v2yic+v#kTdvf1n5=+q1sGlczO$)v6xb zZ<<|wZY8c65K#+}?qq%ePONOocs{E72Z4}6W%+PrR0u|M zVH)AUl)d(a{TkuTn2A1{XnQ%DxSOGXhyHBIy<*lJTTKSR zgXEgSj}wxEE*V#EIPZ{;ZYFIL^10td*hU(zFepfU1C`+pVwH z0mFx!)s?fSD`oT;K=DIQ9xSNU?7@gvt{f5!ks25F{IebcV( zB600YG_#Yb?vc#+eK+-%-8&4}3LduI*rnH6+X_p(xIO+{B4SS;&0{Kj7}be86p7dFaB|V3fhn3Qrymw!Q`wd^(p`YHHk<>K!jZFPKmAci)35~RubNhekY;;OyPn5MI zYWc_p@bg^C@#MJ}pl7Mh^ma@XQKUk7gN>jI@7-007TVYNlFTH$flh072Iha@!)G~j zq1+!gZn4+UD6I;xY1Cu($U^OrBHzkijRo2zBu0F8#}+}N z5ox=LrJC#)s z`@fVeAXR`fTm$>BDci(YH%nY{3}0Pq1^H?J#hZ>Er)Y0eW;G|DO^;R$mz_L)X0&9?ea0xltc8jC8TLPcz&Y4QZ5-@q@;MO1fx$b{!hnDPF z+PbgY>svBiBo%yOXWHd|pl>Sb35w$=Keg~#!n(F;V)@!Lsw#h=-7f3ut~k0fs);FV zBfg=nmfW~BXeQdyJ+YqJqPSFTr&pqCFm9&Cbr;4FK_gwFIcLR|H)O+|FWj<=S%&wq zzo2gZUSDxV8A5jt9v5zhTSu49!TTT(+YhzsK)?AMx5qDbAL2+a;)S^1@Jq5&?w<>1 zQfTeCp|ach8k|y+m=2lkcx60jWCfxocz6n4K$uQ?A$1AWL2{AvI-;E~;{n7hzPDZf zAb<#_Ww@KDkDMU&b_H;=a8kRos;(H#4T;VJYJ>*ax&|h!X&J7FGxWXRTa> zSn!Nf4%^^3D+DkH&YVN5g$u8>-Q115*+!}=S9xs?KTT7{r3O-l@gIha`$mREol_mS z>Q~Bd61UM@Z){bHUJiIjmovtDs5bf|8(N&E+6 zqQlGreu!}3FcWdFdw4Kt0gta7FT-3cVDRMTpJ8?X+k8tME1K%HX!m!wEbUm&zr?8w z9HFWD1ue{)-Q+yQ;K5_N)yvr#_E*PnNzAys7b~Myk}7OAfSFQdd!0>KGE!O9Jhj?L zfTE1l;de=bMC$nZRK#y=E){kcwG5q@<;4>V4R{?K{SSiOlPvI#Pv`@H73(lddC9T`HK4}?zLR9DP zH4j!h9B9I!rmprBu^rr$G}S1Yd%BpB6p7IO46mntyGXb@e-rYtKOigc^(- zSOVgU$8zSKW+}TIJB({4_vAu*-(##Zz<})A4>4#3)phx6oD zSDebR2VeXTzY=P)Y%{9fQUyd)<*~4etwywCjCFAdq0zl(+0ouJV%u;tHd3yyn?E}a z!A3s&lTw`>B!3@4;QT;Z{G}}kS>6B{>u&OQCvyQq)mt4Ej>3;#w%OpW;wLm4pd2C9 zkbr+sDi_%sOjHc*T|l76=|Ys0UB1I}S6mp`O>&+_GmHp+z}aaI9Nd_Y^iBh!&jlL_ z7lA@H=T8)+n#A|pu;xmYx+PB}oDV6<(5ORq3<*OOBG`=o!ZL_~B~M45!aJzJnAw4L zrS?P0??5G049c`jc1o2X;w8Z}jlFmILf!Hn<*o)3dbQAPRagkCaiHiEcukEIfo@-B zBMu22kg;rZ1#tQl>_wDSnGyHHB~uHP?f@fQL2ntnqyug(D~~l%T?=Wc#d)E-fhcn= zgnFT=lzmR*4UZkb7jdOMd1v}L4tnTuQ_ZUF>6%gdkZAe`x1sQaX}PM^vk7U_G=Y+B z$F4FU{iaZ1>WRoM^JahYRTc_Bx;+sXcVMAuJH|Vz zYSdb5&Sxg9@reD*X*7@XL?>-EU+r=q^oITw37Igc9ry%{9{a}YXGoNX~35U7Ntz6(VhR zM2Zd^Q1wx1;5@HN0Wa%h7YichKxs@MOFyMt?qy9;pGy{U#F%1ly6YdoU9)vHAe8M_ zYhvVh#!iXH0WJY^rxuTQ8nQ;l;|03R6~n1 z&gwMa^DTQZvEkH+RmcKXiC5yWU!y0W=9(&1=SQ3}KXt~C(O}^Mt%c3LPtrvq#P`&| zsY*Gw=b0(Zs>fpYkF_`Z`FuV_8y>{=@N2&CymV@s$M`<>+xzJUkxHI|dO1eCS#g@o zekA^v3{tWM7V{$z%#U}8Uc8~e)XVR=l`}OQng%$T(od4;;;i!jJ3sNiIg$U*0#CIW zcU^VyURGMM6rn_&rkC(a{Fso{A(6UxcKgNBjV`SWRpBY0Sv2dL%ERNwYkNSQt z>TVRLdPGqr{mD~H>6c(Zximk`&y9wB9_V4uZZ~WW$9-V-w5zPl<~a)H(5=j$@)dKB z`1h_Wzyh+&`jvnQpS0C@tY;s+&&SLp>-*jWkbQ!-ovGY)97lmayXg5Lj&dS~4t7xF z^F1y!{=17yn>LF~n_G8Ue9|P{(MGF^2ZP8%t*jAPoY23gbb5u0^*>rJtfuHPH zL6q%Ztku^v?3G$UFtH|nu+lJ!8M;bb>=4vKN1LymB?m9VbTWv9&Sgc#%E7QB37RTM zoIXOBGU{mhK}=3iAS%lANzA>hugk6LTCsjnK;%|P+~P5pE>S_6Ivl<^LrkGxouzCB zvn2%=bIyf9g~wsAAjy4qf62DXi*eP%p|hnPcdu#_RkZ|hvuIU9qcob+#6@hx<;6tY z!TSJs7pOSEh0NSWF&(QhxR~nL83R<*V3E|TE6dVN8Z+w)xSf3)iMEx>8!I7Fx3?pb zg6PlCfFea*?K1=KeTyWbV87?5VWmp%e&0huRFTtXO$(zp>h17C%!N2+Pn?xAj5l?2 zlu2Rxv~r|f5PdYio26OXse{IHuhyh5>&x=hsYPHinHOI=r%gJq?zl>O$p*dS zL|U+gSZZz1MJz5@N#uTVAb-1}bPuw7S9<@0nf5+=a+tE?Rx>xcB*?R4o5zoB3QM2y zHbqTGl|atWUq?B6(gv0mqM2$;P+w9s+wW+w+C$Fg^wt6*VqYxoH|pMu`YVMKkyS{o zD%ME5T?`5k_&@HG&yh51G9)Ql+A2%!bwa z?(SdFuXegmK@E8g#(x}fu6Xyb6`I3>t6)WH89pw2jpEQT17a>ZuDk}gs;12RBv2$} z@tkiUYH^WBFRLzd_}Rd7II*zxvj*1!y-3vXDzNecnw9G*Fv-VS>T+gqz>gF~T*3MBWuJ?}pAPR)5^+apr89(Se zu88agZEAB~a-Kf2p(;gBy8K)qOGuE=k%}kjd`KY3P+yfUb+n!;!Tuw*R=ab})KQEWA^pfsk#Aj!5KS=r9T9xb61+Q~G8RV66x>o0`bmN{8HtMX zA%3%w)=wkAVCCSrt^5z_zQfA>f+ufm3xnekw*5dze(7>St`)5cj|WubplemEcU$c)y;>S&*ye(FJxnD5tUrlt?3`%hS8VvyW3PT1~E-XSj0s|u)8Kk z-joSfuD76CO@O$Z)vP6_EX?-?dNp`w%M|M-Rb>_1?LQ$Cs4B>n(RitCJ%J^fO8F|C z%zmmLn9d#QJ6EvC`-b>Fp_!V7lMpb(U9*8k&E=95&9iZXK13lU@KcR3aJLLNP_z@3 zWR5jB5PvxQuKSf=el zj=jBC&$iM|Unj#&C?7IoNu%eqkTH+RPd^qzr*rY*(MjP>b=(PSZNp+7-aP);g2$(f z_&Do@n92DEo_xYXNhO@-CI?OBhNUG{Zs_Dg2m)l$AU!!xP8h)Ea8XAL)Qv}sr)YXN zaGP_G=51Ui;;dl316|eQ#wj~CTh9Q2EvAAZUZlZFZh1cWEBEG~;dsBWd=Q4ALeVg} ze$-zQ@T8UmC1bGHOEPPve|m$@3;3(dnXdoGPujK{8>P`IWqpM9q}|O#0^?Xm zM5J7Gq9BhfODP^2-$H{l;Dv)MB@s(i!(yA9 zXJ+G;PW@EJD?HlVbf)=;W$v=}8rCj&`?M@_oODXHaY=W(=MuTl`>6L_Q{qB-vEnR^ z%N^l3?Z|}8OBUuST^(?!dlFcjOsCo0PX?@ky^9Q_#JBD2|AT z4X!JA8jr1N)>YPpViP*#=^gVpaBxyWapYDhrzDFyLLI*cepH#2w|&>Ct}-l1MB+a2 z{_Q1ET~%QKH!-Fn$pWdaR-@2Ob?-gZ)4=OoEn8OM7Ze(QpPen1HP_iV*!|X0@2$8K z^C{`^Jgu)jfD4W{XAw;zL(S@rukA|KB;{vHp7@1HaWPbt1iN`8|!>M($aN zA|Q(l-OmjxLE43HkWq;WyGJVP;PF*cq)w+;imd z1gSC183c~_-n^P)_|hhVBUSlRW;&7R75ZXojmU4>#X$tB%Ws_U#)9Ql#o)|C*mc@&4r60 z1ssc*(PL{=UPov84|ip?#yj3UV^OwQ*e@3>o9Pa=3Kh4P{~8c)V#wrpTFUYc|u#~ft*rBTzwPtag|-wQrc+HTk4(YmNUi8smDmqk)Y;VlKof<6i18Aju3!NG*y8P<-Z3a=K(ENusxwZ zv+N_-lBUSgS*FuMgn@fG$(}_Ah->MDLHBtd?Ua;b&PFKvb@FJ9i8fdQ*@kokpB8_g zuZbhFCzrZT7A2NS;qme^lF!A7MSqWeHD?YfnOeHA zBo+TJ59B)w#}&pu?fGFQAn~rZmw9PKEa^Z&C6|UDhVz5yW-aBuN#lx?d<;RPHz?Z;`FRPahu3lp`ewrq+9WPa#R+VOpm2FQ zCVB19cx$!C!4C0m*BGIG!+L9$l8e*K``A{D~L3*noB?DY0l4I{fQm62{ zTP{23NWkN@_2BST?MuhnuddHhQK4e2nJVM`EKY}nwEdCDh}EJmim z?b^%Co;<^L?0Bevdv=HoZh$K{%OXU5{bAwl;4OPMh+s)Z~-NnUon|tVt=mC%`e2K@18K(sbB--43-vKwqjo@)VfxdS<%pTbESuPw3!9 z>d1^7gNa=)D78@GX7Do7B-xTP8u&wkgY5Hpcn)JLXOeL~8Y31G+`-gCTmomR(sg{S zq3>LRU=?EAx6c<($OP4=r$9yZHENasG#^S&*(QVdoj0}hm<=uAy@bfpSb*Ss_AT-5 z!sECkMU2c5`hf|h=_88`6Ry?NUZN;b;uvYCtQ`e}hVi?_=og9hKt1R~apEYDuqnQ( z)ps~-%pS! z!(dlBh1ehqSzfpGvUeDTA8zR)Od0rc5%Res)IK3|h)<7T_a;)RypuLAC0H~>k|NDG zu3GnmUU?%%V?ljGHEt!eYORH_jIBbj;@!K{(ydK(KX+2NWM{734}HxNvKuiO?8nRn z4}`+!KpS!AI7(yb=hrt4f20jmwtLz6vFTPa1mL!|Ju^*a&3GR@h@kHzXIdA^gN$8E zCD*J`+u{2yE+{T6Gt<#8eB2~AKLkls9{XkF0yR8ii>njnI!`McTm4eL<=YmP*-uY3 z$CUs*8nP)r9a_#FAvVDLQr#mJ!1ZA$_$o)2j%$Z6Q_d1`er|YhZn=BLn6YX? zanpDO(=kXFb)lfd(qCi>&>n#~%j|4NCbPWToz{az-&KwWxM~bVN$Mdd>WWhfxM(`e zb%RrrE}fs8bFYXoQ@W5B1KzakTEAfkEk9Clj4Wu^z}&jR7RXwaL-@3)Ng0T3$QUgU zR$&$CD2e@L2ledBE_B-zEe!PZ_=379DW_+x3^MHRZTdUU-Hfj-gP>7O0;UFuF{3z+ z_}#O4ZJ)&V{cJNhvY*9K)FN))%dEC?sRRE>yBWF<3pf*46!c+Ra@!h8Z z;S`sBh!IvkElThnDjPRPq9h;pMUAKSG${!xD>93D+4;AoFXIEnTP-9qDnq&e1rEQ1 z-yDAidbte-4)p5~MpFxvGIp$g*1o7j-PB95Vz&|_VCWilFrg0hkI55zrqNs1X@*5M z8mx*Da+4*fbSx6e8`s#JJG5@qIWYZ$p%Wzg2h&~EkS^?+XR<_QrUQ;F>PYxu^#W7- ztZKzcf=Hg17B|Wf)*dIh3qDpD7jmVl>ChVs)1ZV-@i2MMh2}RBG2tN0@iDSyjB=9x z&vSpyN7!Av$8fz1EvS`uQ9$|o!{m3__e8S?9C|KFB~GHrINTvq2q;8st%U}_73zA( z#Y8WRpqPRB)r9dF)m{-EJ!W)Z*58e~FF^R0(P|?7w~f^r;e&zK=8Cv0U%!Tx5x*JT zijIgqV(WaXkI5@C1m>g!(Plo3NZ3z>yranAlK4Pl!&hUc-PYx35nI^htKlrxw;wp?ZwlCiU7xZ!{rs_^euZcUS3lglBbMf z-Mi@PdexlWl|&GJ?B_#y4Pf@6loFu(9XC$DXM+%)GbUanPwby&H-boNmyVQxS5SxV z{cvzLIBwQTvtCl4JQXoMCm1p0!^wG9$7)|Q@vb_85(dT|?VMu(V zT>l4ixSoZ46LO{J@qE03j#W)!X~+?S3jKXN6F?DOFh zRkW07v`dnlT&XGvmB!6V&_div4^P_p#M>^w=oGWh(Z>&nC%x zWp}!{M4||XHD}N)bl(&X`J;=>YxeKS9PaU9j(z8abQMW6CTg}?dmk;$R-CVm6HBt{ zxq?msjH_4PFC{NVPf>r}MvW#MZNZD4`?6hdXSX`Qzcv_%FxiLNveGxuW`id<>`Yhf z{1HW%FVFlww)!g3T*EYprEsFD-tW}r5UFhnnf+;`DryEQ{|~<2r5?gqZG@d~_NADwhpXsM@J^J^ z7xtt#fS<;jYW)a*j#*kI3YQfDN<#l9tb$vyOjS+!*Sk1L{~rwfWv3@4cnUUNj3-&<$xwcv zqnX}8p0fS~NYAi~EPCXe%CS<@W=jPe!||L_V;k$>Wz`c>Fr9R8O^ZU=lqYiOA4&Ul z{UCkccZ>CUl-QTK(R&o9rArE0Rw_NTyD#Khpqy@nr;o_N9FgmCd{#k2!$!I1ETD@^ zGFVBECe@MOYBK+E*4bua%gm>l)ef1k{|EE`+oYV<+Eg4Gc*d{GVX)!a@iKLp29j)g zd16>L-Hc*nct%(L)aO#KtHWal08f%b?hN;J%Xb!IK?Z=j&P!xKo)HLtydXu6nL=Ew0Yudzj^Wq$<1?ALxGNDnaEh}RAIhT+Z?u4! zD-&PK9Cc{sNHPKc;PZma)~GmqSN%j$d&zu!w)Wo@C!x{-eIuFBwis=XH>oLpgIA~Kc3 z3*Qg}by)V~zGE)Bn1rHyx~D;>C_5-cOFGD*?UTO||O03sAZf9$nG<#|Ve&oJ8TgAF}%4lH*WKkV5?HA4QJt^2clQZ=1)PRMEcR8=KkyWkbm}^W~GnHh7 zip>}^O18aL#T=M9)T32AtDbW+LHAvi1aBt;tPJ5ETFd83R%C+M8L*220s`ljN@T&P zFj^-P*&3R3Vfgm5(MK_=XSO9)2`RcW7IK>wg4}iH6)8b&1?{I4->n7Q8c!7R<0sJe z!74n?eke7FCp43)Hn0ArxrEq>RHA^7M6Z=1QRu?{uJ>kx$VetH8R{L?iuuc0Tb;JZ zD|!&4&JB}StNym~d1O}VC4)E^7#-XwCFA@~GDa2j{8Scwus{&CIiEIoa$aZe20+0U zC#8@}QhMUr*xLiyH4#% zqcUzzXra<~qA~c&tyh_|CEdJFwpXsTJ30Nbk;K zJddZ3tz9_g)W`iaNICqNH4!N)A)<-8g1Q6+84}yw8?)k^(*Y4t~e^uW|Jk8p=9*j zL%tPA0zTu$hg}jt3sM1+3!b#2 zjrL8WGlgB6LnKX>b3L(q?kD4?+Hs!848Ap=0^Z#U%bRkVFNIosXH;>gR}~e2?KJ8s zaJR9!G7Vn<5pWI#uQU>AIG#wVJht7uAjRy;NZK5IH4@~8!)5^Yd6VFwmg6GYz>s12 zGE&5Rn|{^+@KalVno%d_xVw{~uH6>IkWfIhJ8Gr~BSb+oS?M~s3CPO?Nb}oYmxQ0| zbdFV7w5b(yVuD6HY$@z0EVKb!^!_5Q0jA{1@ z`Wee|cjiti0}ITOq~owc46nQh<>Z_DZOhl3?aHdB&3p@$(~d1EC`U54mX^N6`SXiL1RtG)0={F-@Ll z-8p34;kzF%S(#`9uV2& zv5r)4_EV#{ekHZ1v!>NeW8EOAMZe&OaUe)DvA?_Cy-DIej>>kyUO21iJPo(wsQ^0P z4u9N_vJphX>HSojij+X-sME{*m(|U)ZX(?}s-y6f?c7dMU){DI|F;0~p-WMYIGc#e zp?Lr5DB@r_CsMz{!I%x7dnhXc_${^+x{+*dvfk7ds*v!W_xamTS}xYO^T5fzlO;py zcOp9-1jv1J*)JHrtm{(Q-?4Pgt*qePkD#6g;eMHo@~H42)0-9iiZ4Rv5wD9WQS&< zQ7QZa85r;WbrQ`>tUP8uCmM?@qF`h4m}L|kOkq2Yi;QsSY2J4Jzhl>#h>3LKR_BZH z%;MGR4;{{RzjD`7Q5amS-}(Dxj`u(Yhh?ofx#c`o1nNvCJm>5FVu>}~3zI9#GCM?l zU`ufvt?xaPwa!DLmS0N)fK}?W4TP`^vg9B0nPJ%?b7CwA9{m)GxlV?~y&CrzXJ&rz ziOW_ZKU=R%9spI;|LS~3%yMVTEH#83|9F%K%)m_!B?(gX6#I-4l47qY%MMDM)S{Xp(`Kj|Zm=jdLrP8AZd=J&toJRcVeYAVsDiLvT?g2Z&xA7ut$UpwE zvQ;8P4tcF79+2G82*l9_-)nx6?8?(6LQ;$&SZWu=zmS|t*TMT)wxsu>^q>AMD3KxM zr>Y9!WsvvMd)rAmCEc9ywVbo6e;=WOI=(MhVhx-E-}7R5pNVj{fIPKEV50SoO`4jQ zX;+ato!?A++V%-jZ*epEfWrHYrw!H$UgaL`1Dp~2&W2CwI0C`Wut7by_*1F-0=SmY z%fo=0y+92_cZD2Zg>dZL6Tzv^znl1tZD5(fM-8O_KL6N9NDK1g5X7u&a|Ja+5!{d5k zyAt}L$|0cVrTlw9L>Jvl-3aA~KiIPMey`$%Zn?t|Iw%(_(WpT2#W~WzhMM3&M~E#$ zVkNYQK$((wLApckr2YgI7XB*WGz|}-P&zj>`H$<_F;QP$bECp zTqH*&REdS?xze-a8=yBvKb!eLDcR*@kqpPbGjekqL;S9Uu=YTMo$+Yd^r5>Zl+ZxS z?s@Z}<9^Zc{^e3@)j;Rwc(mQIwXIb=qmk{K?Qg+PP?($rdYqvU@F6e^0tJt_V^cB- zIo31c2?@L%BZtF<(wW5S8X(OeYM?Dy*R|3R*Loh|rHlJ1oPQPQ1^P1UES=d8si1kD^%|(3V#rV#-|2;)vI`Qk39NN5Ba^KU9Bme(Ym1$lNZxQG zMrPy>7~F)ojd6GgF-2a3Cl2VTPxClL!`nP1!djRd7QtOo6RhvvHU~&^nNQJ!)i2Q= zp521MF+Mx?y91nGELjoDY6OZ+*xiDmfJH2(=&aB4((zHG6I2wk-?&04zNR5(@u>FO zP&nJbw$`ZUe{nm=1?{`dgAB@sz{g%Xjhj4a@|ZPPXkb|9^6m&Xq#$PFV>I`T%n|IR z1u3|FIpJ159cT-iBK5HT#vA_1^8hpA{^S24Ca1rxeB(60ALwWUn0hNAP7{xVU{b%t z^NfRWoR41(jn7SOs7!vtrdl~*>e7MZj1o<+Z#5tA9H%&~zo&V{Fj(uOC2AOncSHW9 zc>=?TPoG$Xv+iCmO*zL26U)^e=G^AobUt5G@0uhtsK@g8il{vm@iu=%e+R`9U+#uQ z_UP21YM+p*JCjxQ<-F^8Xig3BG=)ip-1SLauMw(kWj?zCR>XHNU0R(Y+fwBWIUT#a z9}y33j!RW=5!0aF3I6e!4T>W^Wk^xcAo`i!C zbNxRUrmK`MG%u3b>%UNAzcG?JummWSdiGUvAH-sAVUBD=t zJiH1bQI}Pj#`qj=yr9fA2 zFKAR1Vi+?zQOII6ucOELVtvVN@FF}hmSq?Lal{{4#rEs%g$C%PARWVGy|$^XJTtv= zJ%pK4Anp)~(j7mXC!dfcooUP8N?-w|^hMbz=({rd(t*s4)y?IbT}gO8rGc0{wR*bY zmEzAbfLeL)-Iv@oj3O<4o;K+{-A|b}-|p#(;kYv zpUO3SjouRdJy@jS4RlJe@`{Ia2JRgwP)FJEQUIB9{5H8#Rm{MZ zJilvmcZ<@<25A+DId;S1O@XaLuG6ElXm{)}_u2~qUwd;d5Pb4sbK zA~K5Vky{6xJq(ZHM#H!WsTHGAA1oh`s#Wx?>ILi|VK{2ckCg=0>({bNzs-9@bn3WK z+m)A}GGdUA9fq{#v_*xDv_o(vI5A?{=zOHLdWOW1liXEthEJ#U+EP_(zWQ)7qwB8C z4Sag@GAY#PG$rd5cfpvicrm!T?bWlIqn}k$f9{ zkuw21H$S(r3l!V$dbpP(QyjfHLi_GRw@R1JStiThg{>wY!;h zX9CJzn@h%sM4xJ=iVB9RotjyxGWqS2=D=glqwx1Y)VdAT);C1*Q^*Ps^lGA z4Fjq3{j^2nE_olv(Y6h*6qMQxR+cR2@|f#$xVgk$2yn$HPm`e;w-8?;l7OcajhZaJ zVu_WAvG}o7r9FbpsZ2~@$c2&eQe^8&A3e`>UVUyC`O}ox?Vb=XU94@;^<9C#dYm3o zKYc{#C@LG4j9}6wPD~?ZEMGQ)=eT&PIF1rH^!*@R?B}mU-?=lp4Er5?rzhX@*m3&2 zt8(mPcl+1ImxSUUPI_VSQ8eBI=Y!^zt0kCoL`+>3)h&2rem6t;+7V+ML}Tj@TBVDS z@%QoHkU-2Aod}0=)zJ>hy2r>fB@QsN*@=_~Z=^GNh^jP=EgNI4K}viTr59_U5RD|E zXo`*oE9C{Wz1i1YY|Fyy`K$>VS60=#wtMcC(77FTB5_8S`ld{?I!jO7&vB<+kXgI& zHM(Shs9I=AdpNv1bqc<^+E)KdP3_tMfd*-nM-fh-JvNh`(#Tz*=Oq`F9dh!CfGz;e ze47K$w1Uq0{HzdtqJ##(3~Ux|8>71EpY03?^RP7WD%V+38eq!L_c$2}j$gipOfdHQ~pvaoSv*e2m zH#yRjZUW|ak`@BzQs1PWtuySOXCB^mI#Z2}!W0>>&azQnj zNk~1`Cuojx!+4SC3x@$kk6X;s-?jh27~PiJM-7!?{sQwP&X&LX0t{}x|iX3HagxQ|8Qhg^rg*{b|M>7FRh^)$d|m2 z3{IOnkvP!ETcgL;D^8@~$CEhK;{QH+rSkpkSDR$)8MWKc5@b%`znILzTs%2a96y12 z$(AFe(nXEhV<+j&FYB5e*?dFBYE4%0{4Egq_KwCM3I;ufKe2UeOC$cs(y`aIstYTQ zGP!Zy8qSMwp78OFRNLz>@ll8v8xzN=X+P51eD!4C0Fu%;k`*!I;KgLx(1=pcoJhx? zyGUIIEvogU;xB$ny)3*B1M}LR`;nP%U}Y*vkCTtk-0&yUYnvaK(a4X{Qw^lV)?UL# z(awFHG5=$0wwTID9^ygVewUJAzt7_Hjx77XwecwI_~w)8tL!@J!|3w22?6obMa-Bo1c1_I2xOw+K+!QC(&EN-Mv(Q$>c!KLGOw#kNf-UhrMq_TgMwsNiu@g z9uvhxB~%LT47?$0Ly)4EaD`SrO@&b%&E04zHn64lQ+F8NrafI6-$Yh*kG>f)E>m9NIO*$(4B81s4dH(#Sy9vNiX07gpLTxV#w)33e|)}4DGRvQ^YW@dT|^M z8-Do+gLZQ{JM~Ydy(}mOJ4?cgiZR9Lf2PGoG%q7u zyBk;bg9H>tzV=N%l8Bc8l-ii3)O@cWT;`*)WpMRXkTW%lDIShS&Mq$Yu11B$`{vpy z31gy$?>3`;9fF>4SA)a^SV}Pys|N%k4jO94g^?Jmz;?zmC^kmMt_oTZs2YvqlwwR%5&S23=w;+hovU%V>%f0;)8>TQ{(s)>XW@a4vJ3Y9II*r9$IC$uxwEWW(5vMLVIrzF8jLKE+U z#1B8UTMF5~kFnkR-ifpdYYULFds-zgLxx^Z9*H!Y3^}q@K1OCu_(di!3+y?Vs$F`G zok&T6+7#RxQF_9b5%0({E8W6)^5)bs13OCgN})LJJ@`Qk(|lthW!pQ8bOhJ3Hj8S1 z)!!ExB?%xBCQADg#FBxHhal$^+h@vAaJALeGKO47OsluiF3CUmkRSi+sF%rcv{%Xx z@rh4xX7%-+t+sd7GB^-Ka`r5BqyUB6!D#J2ESaGYYjW&8+VFf8Zv!us&?U2^Br;HE zSa=hYovYww+?>n#d%Vny7=zlKT#D*G(IsuE_(^Xy13 zkKrOlqpv2kBqjJOzWmtMNgle=MKC>?uWA0cc%tw8r=)dI?^hl-Iga;zc$%An?M5dJsMen+)FQzZJd@eAwlVc+}_TF zcV?~I@ASgncGLW;iXJ;U^yS;*njgiJ#X0f;(F3h0G67reNTJp z#zK?dKN!r}jhXBL2-HpB{gyYqn;=wm)&}Rma}p0k4eNQ}uY~Dxzk9UaRb|2dU>LTd`+WPJkqv|R$*?B_p1KL9&J4*n z$k5U}YJ`X8@^)F;D4q(kM>SkeVryu^?xiH~NF-w`dk8#1R7CXB6fARkWU*fWE_iJB zOG~che{YOL|Bk=)R{Vo$8>BoKAw#kMUGO;E>bI{!{o1jE)%92TV&U|p1gd*#S9~FH zNwbGG9|1HAqEAU09wq)=6wsYuLcKk$qKH@ zTh)_{Hx7e{h@U8Ifk-#oi z;qYSCI?F89D6K(|%*GkILDui-L6N+DbcS?rq0@+T10!qx5#F@v69uM$$)y&7MHomy zpek5d%lMiBevH@gO;V>**0P>e*8B=G`$O;lL+w0(+$sOVJsJEg**5J*=!DGBx{xxKGAI(t~dTwFSllPNXrUziW z2_`P^Q1ja*-OB+p^5R6-vIGv;Evc7t`S~q{;fyVhG-lOgf?e8rzo+act+iV!$9^xi znN83&5P7#!X9|y+G*bFG_`W6#J?SOd-fmtg7l>UHT8!plbae#~&IvRyCMCzdF5yi* zDq{!Mn-wMe;4#z4u{^C6*kSSJ`F$PtF^(p!88H`g^5T>x+ExcM`#-lN;7~jy0)OKV zFLOFOS6n=K+_0lm_7{d!%0A$5jm*cxIfRrt8l1p+DHK9!9*y>~YYtM)S z@0k>}oeNJeHhqU@i9TDeH+WYAUuwc7IK6dYL*Lbn2z!sO#(q!S!Q8ZDW&$qA=mY$e z=J%pBfkTWa=l-8y_w?EUqDU9(-xIRxPX1_i_uQ^@LS8I#VITMH8cs+$X?!zQpLCSJJd9uoJw?Ey#55> zVX#Kqzwb9sMGIbY-Hx=T*lo5E^Z2L~U)`h4e_tQE4H6Ls&HO!Bet1_>>F>-({<;IpZCl4%jiuu$>LJsRg953E>}xSs8wW@vPCC~OF%eN#IOlCnsl_l{mJ6rpPwMZM#nsf5 z)*RWNXl0A36<_E-#2_T`*(*My1-o?Q z0&{l=Waq0Ysm;fzeMs?116nN0lCC-~+Ddf4bRHa+*$EYpcnB{#W@Jqek4gYNc{^{z ztSb_+4pnr*un*S3v&g^?0x#rEE|F|XoJhCRIVj2aG9|wGRCy*^hArAWHWo!WPBvS_ zTV~(b$$MV6Zzw@EM*mc9*xhnKmfuPK<~NDs!#L%Z6G?kA0?SjYSdG89vlWqLJ&`p=}Jjttfw@~@|D zKpCF6d@Z9mNb`}|3v|7sP|=xeV)-nj($JN7W&xW$4#_%8y^TNPG_#?#Y1X2==o$uy zaH|6UO~n0y{QlC^pNx4ckname0QEdDE_6`{{S_j-pf?Ffc>#sl_;2hdbQen3u>sRG zjDL=V(@{stI62PKW;%HAC&i_dD}tOu%9d&@cW0WL$zy^aIQZex3Y1=LdTd_%{%qaG zjtZWBX>n^DLOn6(Y6FeBi6UljaL#1N6rG3A+$8{B#6!|Lx47ZO+ z(stKIEB3LF5@xpM{z7{vK6r1<)wbO-nU)~%fG{~=f z_HmVZC(IL6G#pRhKWkhh4tz19Or@87310(K2*&A0)>JeUc52X1kpN}2{)78ttnkfR zcL-c)41inyHPMo)Rr5jsh=YcV#_!MB2>2YtTeR;{MVZUy+ffgGW6q1TDfkc)>cG4b zC%ub~%&1N5iG`T%xhXQ3meq8P@ch-U@@3uBX)JQyRrGDYH|#CrCE)EJ40>SPlVE_Q z`#+ef9=&bidz68509ULf^5y2Ce^ly5BQ?i3MnLIozeticadfak$(;Aet|)x;2Sz{L zuOY7WmD}KAwqIq}Aj2 zWe&^pJVJuILVlO5LJ4Lr3S7p;*?GuUbQ5s&R9t~C{#oNvC#%*yr|HE(?SiDV-IllZ zQKH+D>w1C868@fpr<8LDJCm}3iBQ{*AN*0EC(wd9sTwD#0N&s zWlgN^I5dNegc;H~WrpZ`tEGF9^iir=p&#y%4!~?uLgMJbVt#{={nQ+VJbCc_X+Q!FmI2KLvVwrKx;zt(;w-3o0&-2&&FhJ?l4gp5#@oY+UrfwTrL4xHEw5))WnnJRIXGur&N zsk~tu7yt=;?4)G2k704D54yWq8^jT=j}$J#%arxX=U5iPqwRVHs8xZ26aonMytvdD z7X+y7sh0D$1lV;b7fKZh=M#>iiT<|7FUN(fEw!f-bk!{QKx+}nx|ZK5%6b_sB5f&h zFASbd3z0SV2XG;hO=mgjoCa}nJpoCOR$fo0(j_3@;$#g&B7@0wWQY(VpxJmmnwyGg zc&vkL_>AYH`zjqHQR~fp?&jb561%@o$bI|*Tw_QNsn1yVS{DxwuN=?p2hlEV8YE^3 zoo`-CX$lT%zC)fgm%N-v+Iiy?*#v;=oMJafluqU9f=%^J$T^+DL}HCDDp0_6FGUo{ zNM~&-I}!e<=5N)7af1&1mQT?XuyqsU_)XJ3;J?qP@;?}-Na2Zi5GT3fAK8jGiL|ja z0cI_#{a0|>3LOzs#!XkhqXn$Xnj?HhAH}GyF(lFXz&B0f z2wr<)RwwoaaJvz+LFjAkM8~al*20hSE#@Ad*ZLt)8O7#fqQ^yxA8N`yVxi(pHtXeA zzC$fQ$nJ6@9kr)C8cWS8@lK`u4umsaGX>YcHJrFqDxSr^^R&f2s*Op$Z>h*T$>XY; zA1x7x@k?PqFQ>`06T{fEMg`(2ZKuS zHuK!)bgIR0=@{4b#Vm{;ZfW%rY>y&2y;pS$JflC?>ulm7NlvPpHgjWKc(E>4oR;1) zFxwetNz05~%KCoNd7-7jUwv{#_lNSlVg2gr(d>^=|F#iw-@VT&f$&pM_Ksq8 zNhhsUmMO$~^hTQ%hl2jF?1>U%3 zXTkqs>n)?&jKXbQC=@B~?he7--Q67u1a~b?ad&r$y9C!34esvlQoI!D&EDtid(YnY z{K%K&N5;tbWUckSYtCoNhVx7RwF*VxXb2GOUK6NtDg`9s&n1+BDx)5!$Kg{gfJ8OcDQfj#rU- zyhDe|UvoJT19(HM*wEC*i;Jp;j+xB z*2TV50U}4t)zSdUZp*W03iQS&W^bM^)_H_Gqxl}>v65XO)ghYgChIkr3$W6rjw!}N z6DP<%Z9ruUlM7kyQ-#4I5PONaiB-9k61SUk*~4A*_}_XMv`T9)fhM|()5~nn$^y6` zyGRSCI)$d+Pbj=l9>ScS$B-<8s5n(TZ$t%Iqh(I!_@!`x1aS#u+zGkS3I{{;)q^S{ zhZaVW0{9_Zb*rSAtR z0k~*&#>C^F@sI~-rB!J{$M-;f;pGQ%`ze6wKJjA#e*u@Admfg(1S}1qy}UY0pkq3j zFRfNjcmhjptw-U4u8KitBd9lN{%S17Il~gH-ZB`&!YlA~vfBt@+gC)Mgx<5GmO@MI zi+K=f)oHUPW~q+OZrx$1m{EK~{OebP(EroOmjIK#cvqb~YGKT9nH#lt!2UQs;S%Oj zy!S-}yNUkxtrZ8~1MxFHOZW{85pmbSM69b@oIHoVP9nPgXI7O9oI9r1qS?+u;QP1Z zcXcqw?(En@^`EddVhM>kU8u3q+r?7gtP!vzjF__0%gtK+68Xo4Mu6DkH~S-aA$g|x zO7jK?4qQy`kb83D0Hf-FJ4Qdwl{#T`SV6cpaO?}HihNkr0B_@cx@A06UCK9{eq+T6 znBvG{QmJp}5gp{~z7@Ip8vhRp-+}Y3{-Ypq{b9pzM6mR#;{5M>{KTu(H64DwRkEuM zx{@kB2d!13_ZwHqy+A6vWOz z9G2)9Us8%j5JVPluS2cFhq%XEANpPEpwlo_;WK{MGYHS;8;T(xgF7~Q=ijSY?Imq> zCYH9Ux4EBVSm2Fse$bl;8SCet)M_)rLJ3B0Ygb4|Y1B78Ot5<}@9g3P78e^A*g4En zsN(_or!Ae?6fA4n-1-7`y@?Da>LN5-^FJBcqmij%gByLtb7jj5PA=8XvLX_QsbfYP&I0I#Aryh}IBw@t&ys~+Ec4%y>x5-ARo+YEBeVDcsWy&*AF5g zwHGz^wx*X!JdS58C(V&E&BxK3bmo;~XJH)W8%h&cxt^>~gO(>kj?-%KSkqRBZ}}xo zk*G!A?0net*TQe1;4%=r>HKHVtnla?B=YInSHcPsF?=sW;{6SCIIc@&ERB+lQ5(r> z_C@E)oFl|rmy2FO9v?D{VnBI^BxfDn#C>V{0sjp=VALJXf<02H*7W*3Y${Bvd;y}4 z>UMbxhQG%3etA8KHQC!(soyjyiB#F*QUW%FEQgd3|6S+-)-it#kxY&fG`Cl=BsD#_be`g5V75e2 z!_x&cIp;h6dhOKO4*4HF{0Ac}vJ@~|BN-q#?@=qaikBFEdK>zZxvA3Rn?)kp!e`h! z6g+9lz22=)$Sa*dW|xVruLDu9vUQ0zAVJuBk?C-Mhu6=G2}7<{n^%pewB%v-_x!fA z6p~ZsK5W-YDN4JbP0z1zZIX#y&UnNGaaC(~vPhL>=e3jGKZ*-|Usw7cuy*-nS$V67 z!y`IRTP&j&00(Y>!Au0Q#8%qJ4#~uWIfMWxVHO6OOLO^E1$<_4R1ayBno@c!bnusMClo|ViL>S@c#LPHa z5*UOQt{exH#GVxSbf9w9F`OigiXG9OCv{Tpc{dvpm-m=!($&s5`)Sm^CFxHKT&Nr5 zi^HZ_z6)8!>P_-Uj%=Y6g09nreicvU}p`LDs30VDsQ;&=!?bc0l- z15e6dO@xermeVyYILkOQnDj3W?dk4-Tv26<1+>tODTg$zCCQ{TPv@N!H4|}ogoTSIQ_enEH z@XMOed(E$=Ylop%q-5&O{^*JqIJ`JzDU?HbXbZZgxsJDKskMofWb2hJdj|aQ0{A2t zZ|eAfF^($lRTV&g0mXaM)ZOJL=Is5B1vvARJKV~dyQwu<(nHHV_Am$aHI7j7WCrCE z8fzT!FO5(bc%N4Lc&;XW!u-qYhuO%-O?m^PY$*5+)3n z8yLZ(f3*4SgIptg$u`EnXQpMcS28&|GFVJCoocg2SGFcy5--QH2NkO6Sb@h7bht7* zec+|sxGHG(@Nw$#FyxFJwX!xsmogeGh7eAKpL@6&heZ*Uu7K1G=b5alPaNm47Y@033UL3M0GOv+CO2Tn%e0Y zdos9X8svcPZ7ti44K&(?*Hz-~jwMd9a2~B;CtR_7x?zywrb5Sz zBfLoQlIhf|ULBatJZ~avCmbnF zB=m^H|H~Y=i8qp1R+oN(hU=)5Yd1bz^1^C;9Xyh||ECF~8R=_st)Y{<)<4D1|Om zOln5QXkw-kLN#NQYcLpaIbNzLgpJNIF5-V#dsZb?G-nIpS?NW;U`O5Jg-??TnC%ZK z=5XnSOp~Pn)Iu}SqLTghz9pKlG-om~g4%aHIz6DNl15(RXQins;)2zty^x4miZq<1 zw{?VUgK5j5QhH-fjsCD(wbqzh5HIPdZY)=JiECXL`jD@M4ym>ySgGa?-%8ODSQ**j zzm6ou6(#6{)fCH-zHwV zn{*@p@?&VF*u}RXxY*1`F0mW5ZoJHK^6(aBabQ8G*W!^lzr~^QyVw#*!qdm4(&KdX zMH&0qTX{l(-)D%)j~IB;jNk@VO2dHI2Cs8Vy$P1PK|D83{JoR;*!_p|_Ayl)f_JY9k8Z^@XaBhGZr)=@X^}s(DsXNQ4Y3 z0|P@rQ!sS$j~SDqwc9%Y)k!R=|B#@iHsw{3xYR_a)0&h_qW9ecQ8gCLA$JM~!B67A ztnr)zrHQoh*)1N1GGJmx|q+?G?f(e#+$(ssJO|L$z|9b zDPEmrFfTPTF2)qcj?L#-Y96sBAmTvv3@#Pc&=F}7JK6RKlxfc)KE+22|L}(V0$)K= zOul~4GOA=+vDp@qlCN|T7}5sWpzOehlWwx6XQB_bRqGdHNNC(^K~gpZn&l9k^y$HH zbK}=Sc2uVKI6)!Wfwd&{NERO$Y+J1%kAfnBYxhV)8D1GsW06J)cZ^1F)&=|Cy`rw{ zXT$HgV(PiF13?v^FJIJmwRS-ceK}j6_GfkW2LEA-f*cykdzNa-P?PQha~{8S7*v5Y zM;|m9Z-jzqKP98aaN6=%hv_S`%g0Fwdycb4+w&eFFi9(JFF$iS)eK zuGL*fi@C~okZq@VTm&ENyp4gJ*faCvZ8aju1NyW! z{9aSjY067~_;nlTQ1f^=Tm2R5IC>43@oCaq!#>oUt!UGJlHoZgQxWvN8LY0UV>QFB z!y-5#^s(XoosCG6PD^44jcF73HA_K$M3H9AI=4(JGpZyc*^NRy@{JuRg9ccYn`C&E zXD^?j#Ax4V>xZBF(_Lz^+6a|sIu5)s`GVsogK9iF!*7fPq5nUlciyP@HNBF#{yad5 z2uk1i&orve?U)DWEVdO%{lVI|vUJL(x2+Zj>-%U1vk;vmX?Eg4a`;ek5rm^x%8X@T z5^I)vT`9tYp8%Swo~$(E%lLr0mxQB91n3V0HBY(RJN=!M#{DI*(uUkP&}<| z(D@7*oRBdT>C!@nFAFTsIeLWE_}FagTQ;&A*1cza7z#KSh?mYJrp9fm$5mC_U) z$KoP2LJ-^kl`zg`Fzv>V{FB>qOsnxMTdF`Z<%AY(^pJWxN5G{Mk)4-yJc~vC8#Q8z z|DH6lkpJVrbzVcj1xww3AXm3Dv&}F6K}FNq|7b&E<{DrZvd?W2xR*Hknl1UL4qiPQ zDuo`dw5=Zc!Eq)bOJyai6}~qWuPuA23E;o#k0V0yzPky*{Tq-fPib85*`~up&Pzah z;lu64aT@1evpcgna@?6if|3PZ@#6QI#zh1~k@DlD#HcGm#MT6i=*6fJ7+y=ZjrTOk zqjFyDSaCmvEHB;ot2;U`{b4|fMq#jb;`F1sq+a+M6Wc=;sk%r6u9j=$A*D{lBMmGD zb?DUrC`-Dryq2qjL66_~98Q(|LYqjtj{&ZY;ibrK$aM6C>9~WMm7CY13D6@yK+>CS zKHLD34)8q3v_3){^p8}JmNhSPO7J0gOc4AJ%JUUz10Qwm{F6&hqV6A%aP;x&wj_O` z1*nN|KyEY!(pME`;+Palvi&P>)a*9aI~g72(N8$n{8WGAw#W&Q=^N9^9dcZkb-g8z z&k&@JD%i6K=w+WF%8tY(>$9)$D(fC0vK-wHJBVW^waOw@c~(*9cx_z*+R~=|oueZ? zZV(!6p5YFlk++2H*4A_lp`(qs@z*>T*dL)?ikWAiI8dN)rGh2yXfQ{rU-B^K*4LRfWX!qRBriLy4>{eD z#?;5r>{`OqreSV-SOi@-gI;Yt+aWyLo}d4ql6Vu~-mgu~^^x4TQPS{Oqvp``@)5cps7;=6YK2eAcN)*N-IachCSbIi)xLtX&Ofd^x0_ajarR2 zKuo;XVP4g5@*fDs#QI~!&W*&+N_jSxh4?*{AImbT{`0uW>bMkzwIm9nxA1(%7baJ9 z3yGAbG2#dk3K*8C0#4lY;kel35$f9kX9+or6m;h#TI}CcRJsmQ;>oZ%ZRCKgXXfoj z|1Ee&Jb8~)jjd;gI(y6XhPk@7<)3I3dk>oDZz6>o5BCh0%==^;N;^#Al${|IalbQI zI09;HkF|d?9!!SR(sEk1chKvHL6pT>ho!@YCGKLuMAgDI2dMg#TC)^*B3y$yOb|v# z*-wMCinp`}7tx#lukM)4oC;K=@?67h8O)bM>5K}V$InY6)^}^jNPtV8;2iPCV@Udf zQ2dNc>ZHJdhe|@uI9JUY*BCvLgJC@3BC%kvLTVd&xHdbz`=nm*+iJ=7@9(p{(>@Bk zhUPp;S`kbrDY2-_8JfedKE!}N6|*9k+NhpUsS>&pyIFOYn#9CZ9DOEEnKR5Mc?2}XvKMh@nCsy^&{8-@ zNJ`Ky>_~r67hUqUz(Kqu9rg)Z)>qDYr6bf8uA@_z@y;Z04h9ot+I+I$4{! zar2>c79Zm>Em@d@hcKjy=nlR(nshl!LjB?phsmex61E+q7w0Q>cPra3q8)Qef63nO zWQ4-T?w1Bs4h>)H75AE;&z~nbTBAzJxaRRi{&7GZGVD@-Zi|olmb(&i5dURxo=v$D z7Zp#vDB@62z(f5GX5J|k#9}1kYM!90ZCJ`#frjri+9+z~d*5WJfJe6HVBFv?t{Y;8KufwWlOOVXsbn$WJ8joR1!j4~7hTc;1uv zp(Rxb5S@w}wzyHbf@KkS)iExOyL+**7ZugmXx8hh1(s^t$LexS8?vad&HRpWV=9>} zc`f+Z_&n#4B$i$jqFZ|V&p$$|<=^$^e0giY+$z=EBzV(0+a(bPgt8s;dUGMkXT*Nm zZP>doyQ6}bHUv$4DJE~~-;eYFTfYZF$#}(2?~pUaH+XF*k3eI!`I#F_LU+R7hjpwS zP;y;DB3a_-ve`-Hhjqssh$QjAwWBTz>O={8lqUmMOfI=f+V3THa-L+`y@X5m3QQM> z5hzH2=JX7gJxe1#2u_}*Yuu!I0#Ns;uAQN+u;m+56|iDboVn%s{2xUGq+l@7(;RkF zr24$AQW%TiKWmoCmP*`vwTC21XxRgXIjxHku^yDQO6BW*_ z%NG1l1uA-e^j05vJQ@^HyLZUp47m1K)A?Wvv_a<@(Tkk!FK_R|?~PmW?@KKofY-u>1rwv!p}|(+iERxc<&dXtI?*Yc zeKoepM6^-OFY41`5--syU!!GAah=bvcEB5I3ak;Od->CAe8vN|{3V`@>F#Pr&;C>{ zPih40TMowxXyj&5W$3l8i58mAwe^mqqMV6iDg*+soMrW(JEzbD@-a7sW_hdH2Dezt zP1^Gtn_U~;|E`$+Z{J`5ET^xBSFm`NuV%4qPczdlsO-P8t*k@L)QseA*(i*{M^fA= zG}8)eTI2*-*-{#$NAu{rW9S@NZRq{H%;C_iBH_ONq$1sJd@=#Qu#M~iYez{bDD9L+ z028pOqKR}NEKM@uRBpj_hK&I_QhIzYDXb$ufm)I`+AItQ8cK$&*4f|YqBy)rufV>! zz*F%*8rei<98M;|CXV@rZCMt8NDe*|5}pJh&=Dc-C)+YSY2LUMow#u}M&$T96a;$V zQoB7N_H-UHfLLPzMu?o7iOv`Yt$L*rCChY!QXb&EoxuXFMYJIriINK zcU7;GYJay@a+H8$xE#W#ZB{lgyVCzbi57PJY$Iyg>1z8qTAvo zlJ~s4a7^CUP_J{OW*BWp%jzr!&zxMdUPW9wU)NRKT5o<7~<8+UEe6c@m^pOE5U>>aj({0^_%=7P4jobO%J)o)GS*dC24qX{R=qM z=I*~_vcbje(G89pgPqy;7OkQ79A`$iM9@)qGwS>;Pi=jp?4cI4-uI+^vFtp{@tE$Y zECZ@$e#1X)&cq`|mR2hOS8SGa?ZwAVA6R^T^vrFqx`nl&qYUBTi`+?(1M2GWL(2BU zdwx#rDT4kr41>cf8gdoJ&oBE9k$Ts=q3M{NHIegTaA6NoyZn+30F~`;B`RJeQ0E!d zR#^DB)c2;p9H@)U5^qWn58I?!Mmzn8dE;?7&R1-yFraveA=@}3D2DWJW9=-lEO&Fk z3zq&SR=LN$k#3>N*UOV1J*47Jc*0fc$U>adtK&z-8u~q3>zE$wCuXl!T)UkKZtDl~BZu-N_q%xu^`)Ac)@;6BP>!=aNiq~jD9@Pf zr+d_hyHzk#WMErfweevRS6SP~kgkt}0&Fwwu^j8oRm6|t1L zF3X(yULl%R(1fjFeOF5vgTZpzaTHN+r5~5zkg7i0{U(EuB%|5mSAAVNi}9$s1q7wf zd9U>2qoXRp%8H|{N+|Nr;+w;Rmu*|z(D-CV~Q zuR)5=N!Hf059;KBJ7CU^<0$LuF+73DKvNX~7h|kM@h^VxDPJP19}@z%TsH*cWSpu# z#xp6KgX1l+xA}u8+PT0!NxT%gNkz@IVUGA4KlS*r8ExckbKY{xS(>YMeUo|Ah&@{( z54AK1FRn3mDZKPnLQ;VXh&O$Zz(+9e!}m>g)vCvf$5gwuL3ph7hR`f8=o4-HpI&<@ zvV5Yk#y@XERq{=;ULrl=QYJ?j7*EU&R=@12jaV)Gc2X&ihJ6nT#Dqsi4#`UyZnix+ zim)grS1DzHu35K+!@c5PGn=hw>OSE zO;iy?hzO6BETYqg@4q=*-1BmEAtq9>+op;Q&3DS2W5(sl#7)K+Jjk;vuYxDIT71l$ zwsr6kus|ijB^WMcr78LZ>c1@JcBu;R`MmeO*(74(vn8K)1gcfZUK4?c`C~n|g}u6% zb1AFG*DU+HbTX{Q;3jNG${idU^^)B3{z29LA>)yk_+HZW35|Tb1H>f20bnNU@cJ7f zn=EuxbS~_5v_()HPu!vJY*n_!KFxiROQP%wZ?aAm?Is7Wt`r;FvZBYn_!sb^6WbfF zY{n;pMws^9yBQvJwf2<1KM~BQo!YL9t>60G<^tkwQtW05p4qU@RJMPraTxK9kHfW- z4)c8FIGTk{@ZFNg;Er#aOtGRji(zw_d+v1J`7BKmRO586Ah)InQG|U(q0zq;jlcPnT#M_}yaJ8chBW>>I8a&cuq>spe95L9i(%~X zSgW>Ub-c!tCNwGQuo|<_pa;giiI~7n8t7wl>G0CW$wdeuktfKB_ZgWL$x0?mH$l_F zV%i=0us7ru-Nr6ow0guSmIY*lFZb99f-VgOo}3dNRV|v|a?P4d2Bh5-HRO(@YuW5z zEpYccKQ!PegK=zfal*T4_qj^Fr24l=V_2xu>+x(QeuVT|my&S+Z#f5j6nr)#P{+~# z0HQtw`qcsReyqw_O2ceGO5QdsSep)he_?qK5 zLCqG&X{$PH#e6P@DTx}S(B}eD!Y>aO5(yG@4RYYM`042w6f1Ia^X_{$V&a$;L|0P) z(J2a*Ac*vlI7XuEgR?QZ1vBH(Ms-z6Il?G`1tp#z#~HrR!-(oR05?CiRC$-X2_8zr?*(s?U z;oKsY&7!u1(IKq0_)EE8c2jH*5jB0}qc-BTH_xm)M=A$r+GC2??*~E)c%uB&!@W*^{=Omx z7G$--_cGTGa10w&hz6Wc)SW!g;-ZB!o2DBmt2bMQru#FUum7M-db$(i5*-8s+Pf@! zR~!zb~wo>_K2d=i$$I7MY)adf+H3mus2Rc;u?wJUJ5 z0t7HRxRV}-6p(lQgU}-8Oy=mE*eBD|1kJt=L7rzR@v_i}!!u$+ltL&)gP{!n8sAhP z!rLeme0go#tcO4oR27C+7S@!^XCO#5X~iWQHHa;*ZR@Qc+L18{lV=_46XPr~2jkj( z?6T7rF1khC8Vnfqy7(L_-wP(Kg(tgeNJ^2X6%n|mM`aAuZrwp=#Q8L$_=JS(z;WlT z^_@q@-K?y*20C=S0tVQ)5|8<{qgm@4aie_~lY0bfq=*5S*;yN|nnq12GLeoi#~m;N zF>$-0iH77;5o8Vl<8blaL_9QO*MAa9r^Y&}rOZiso+4-|E$ z=z=SQl!be*5*$twy`JH@F%pl6PfEm}&^;vdsco)T2v{~%zgzeu0G?_63a%=;sIRH0 zI}ok+dnfghny&IX>J$KH`-@(}J2$Qs2?TG3zBdi0MbVUV0S zmquc&wR<5ZD$zy#W@Xy5e^6@q{Pulet%hc!f0n)M5uBqft0ug9R90|T)X;K@xDs@6 zSeI<9_+rxH$)zU@UUR9kp&7}E?r_W(%i;V6e4y;PnKRH+ZA`Bx5y{FJx-U ziqSaQe}4?Vi0H2Wy6hzl9sP2EtR4wUC5{$!8gGf0Mo^(*ff^cj$`fw8FI0PR|KE2*<=3* zQn(g{8~ORE2gfq88P9d6Sn5u{PutC}$hTdiSX#m#lHh zC41;9RG{lKR_b&NM701NxgvA1M4l>9ETPzJGa*eKqSQ zd9AU3(|;BXx>(7qSgAj3+ij0K*18P4YwU!Z-Fy0cTVyN$HKqgCB^S94aY*8hIE|+M z5~!dx71s5F6($@aHeP_gd>2Z>=_cJ=+a;1XdmPm@T=6O-J9r?p$HON{&Oh-&S#h_C zMo^0Tck0Odqw|6Ym~j#^@}B=is3(q;%Uq0@JHB_DYpIJK0r&9?dMZC$;bKSDHLx&_ z)57*y#BTks=y2h}tLIURkcyx8w(;m%KE?2e0{E1#jGMyHDK8V5G!kk%!5N`wh3zKA1XWvco0cssoN8d0OjVdul0s0AW^(a z4V(|9qWm0-;pjvt99O=iuoOm)avWF2){2a zM#)+S{W?LHZS#*EKX%bvHAASu{W;8?D+N zFXMgS_1Pxs+85uwyU$k+BEK)_CxhDvkofv5IupsvU!3?SSQmsb0Wj2xl2QoWk5-jz z-m{}=Ulxab94|6l@;G#wQ$^u3UsuLLADRG|qu)I5I^!G$l~dVkjww0$v2H>K(^x(>m-= zPj>o!b+Kiu%KX0!YO3(La0KJ=-y-#&aBKh(S#rl!(v7p1S-Ff2*xd}n98cc4IUzmd z;p1vXA){%N(6^IgtJ4*-^y)6Um@oOsYT8U6iL<+Q>`lF+SHFS?-qcqgd~ASzHtyuV zvlbUC<)qc{VUqzrc!ocNuz}3e4L}~P+c23i7@)oRuc3VU^IHo;E<65i1E#l+!2OR^ zCvf2T`adY#fSDbLcCXc|Q1!|CTjVkHK%z_7OP|R|di+t<6PFtIL$1{Adp4K`s7n_w z%MhTz)#JdZfd7(=$NTKTq>EH)WKU-?#%rIy0;61rc|G?G?{Cp<5ykV9s%tWKH*s;? z6lIJfH=vum!#T}VWgR=7m_D+Fo0eE&c98_OS4S&wJ}jTJcRn7&CWV*m2u_HS?OF{A z?Cb8@!S!8*Zq(s-93zq-`FI_)B5;;NU;1#!A;7Y}Ny8(Nh87{4u1;Oemz88wB;h)6 z>fUPQAiO$msA$M4;R-A%*zBs-aL|1os+a!?nx=m+NF)&w+PgKounq26>n;I zIArk#a-+mj+Zu#ttmF^MZA-;@>eWGbjEcwdk!}&{7fb|lO^}Zs?(iNWwWu+#|f`sWfV-a-@Nj_Y3|1?KJQC`};X>1f-lV z_}x7Xj>=d8vP*7XE+bI5Gbk^%&d#ZtxSAPIv62f=+{Vk)fqKj#DS-oX!z^F+kH-bO zo+cM1+3IanA*AR!pRE(+{IVq>GdhW_AJM=h!hcYHPp3$#jF|18RKE5_fx_~*ng)r= zhH{|fx^r_8^}=vqPE*{@Ix5`~bKEMT%%FK)eBu$b zwBeB~ZWb#Hb8O?W!o;t`Mh;~x^U9S&6GqFyDUABc(U6a34XfahxtyR`wB{f|#y*D=5zRqI8%UcQ1fN>$|AZ#6cqc8|5gX^59LQY(=uP=s=KwVTi()9s|$G#*~>I}9~kg6F{LZW zw?9|Lz5 zE5)pTT01^flNZK+3Yv<|5I)`Mnbh653}Oa{e?7^$$aI-6L6L7-&_`*m%v}Yh4@`=U z4NT<@?a6oe@5bWQDt`7MF~5z8Dgq&)+q4qvC!w)2?EZs-Xx$38P0@o;X^2q>Amz+w z3%|-cWpJ+iNUNsF!Z+q=6Om>AZOz>7j$EZ+(U{~+d|Bz+p zO!QlvI<)!GyVdYbQgZm3QhLW)ZUYvvZ8V!{HcO}KV)#9(Vb=Jxd8QWBJ#X5gX)Mo8 z9b1c0CGW#gX+4?o;(Q-u6x9UT zeX>b*t6yca5r{iiD(Xx)!u8(^stq#T32-=jLkDl7p*`P$SX&G5Zo2~B)?cirFs|E; zpJnTeQrD1HCu?veAKwBcWX*RI&^-ZkiAUZbT+AR=3n{(B$I-C^QDOnUWTwyn6 zq;?x6Im!4hk!0~`tfBLCgZoumpBWwk?VefhIaaRWuTXqOyQIc|CB2U15+cdGXCZ<1 zc3P;GzwJqwY{21R?7!d%kZ6z6(e?ivTTODCpFbM%GKyo@dbJEbbRIS&cAL*=#M@@r zM=QHcAV}>@-QK}~2-logIbldZ8%LE%g7=@HOaNaMob>sDbF<({Meg2VOxn;PEUP{> z{}#L`T`&d4i|~o*t(*y5Eo(3=ZCyPgJhr$WRxuCYkDsjFdV0HQK$SsBm{#B7Rj)K_ zSz1@d{KZXI+5<%Xb=#UCn^m7%E+)z8kqN3+F^Dza@~8>Oqzlm0#|CK30P~;qzcqCg zj8D|J_2uZaG;H4f3P!zs9UKl2lWJls;gh}|JnET0ZNg9BbtP{@!;zrDGLE(7%6nKF z(y>Y9MJ~W0oRsr&oo9{W_jHRff*>b_+S)$e^V`=eLjt)F|3UGv1c>JGK3-ki-{|N6 zg9`7sNYmMMqFKRx+TR{+MZ>#Fph9-yB+P50q%T#FSdU7HnxwqZLS&9J+T&WV(Jr!J z&f_w-49{qypB^$UF&9~LL-b;ujMMx;mi>9G{_}LRUNl;Fs5wI9)erhbiFE?-TQnYG z#K7$ASVji_@J%;(&}$Vh)@6bi`rX=Tw}&n}6>F6RnF1W^mLQ?)bpf~=3PIKoWK3Oy zPl0$xMdjKtb-$N=O?=K2KPqK{J`ENVpq*IqM^^7`Hc*H-+}1Bj>Tg+T#gQuk`rK{* zcl&em!8BTjjzYb}W>H|h75s6{ZTYUkyXrDbtSDc8B?R7_hdL%Zm7m#8bG4;CfF-*D zcv`|(Th!9$dQTV5Kcfq5rm~AMbS}RDbOP)8&NbGnESU_eeBPzkiA}rL!BJj`80A)% z*13Dj^1qtWzZ4C|rPP4?l|uP=j)-|n=HMdIQ171VN#p$d6!THM|F}m?XAC&!QhT5H zD;OX?H)VY2y0Lz;_yC$0{x%o>2gL!gd&rzTFVB64_hXO)RMcm$=mBsU3O; zV;7vZC;sEyY~0iVn%7LfRh({_1Z0YeGDBb$Am>Z}nLnK~pJIGsr_RX;$FZV5ux8u! zWw9fYgo)rGaF`I+c-5&-oL!5U$=x9|VdcCs>h!I#{KVXg;d?}ICQ}+w$kZi29J@V2ir$DDCIBSFQUI=t2GSE!26xMU1t z7h~2L=A(<4W8e2=eiT~_WrF!}hFX4nbz)%*e7clELX8Q|p_FQ(Q2-%gVg&JcPXXuF z6$E-dVRCAQ*iv%it@fV)2cRL;AkS#91jfprj(d1~w#P^b>Annmu|Qnb_DY zY13|OacZ(Sx8E2UsCc9=uqK?^iW~g$+^K!zzPfxcRXZ^gb)rddxgSm)Cy(|gxrLRD zNAEkoJZlD%T|Sw;%Yb}+0{q}Z^+sK|Ob}Q^@tSz$%~)i$;vbkT zPL0+sf|#i{B48?WY&o!n1#2H8XSVB?Vx1keamVQQm-0KDmbf!jcRJBY-u#vG!`^q1 z{y`;*{boFTo>r=ZghmL~sb9nEoekF#3vb{73hKv~oJ+LxnRQ<5L~4Sk?ef%maJmQ#E&*viT73;)Kv;Q z#J1SfwRCN06)UcmrD<$9_iZ?r^^K=>oon>P)|~xq$i}aD!%qyFED4(H{7cG5Y`uzl zWq#e&O2vFNZA|Pv#EBD&V@NgU<>6r`OPxwYgdH)GrE?>xgCj`KL9G#Px}7vf->DY* zsA;oiSfywh#6j-2uRLH$9F z<>;_WlRr%M@4u3zw7abt49N$dztV2$u_+}blM6f85Tj`QC7KmD|CH6B2Ybh5Iy|c@^f5tg_APKTH|I98Wk>y z#M>v>;(c&qlDB3pR~5njbn}+wb1bOr+aZp)}1g-+zQq&Qj`P*6-=cs z^ATthMzGC$Na>hyo|3ZtUMrs4a(vHK$YZc8QO7-LtCItPlnWF}a3Y0F(ff_7X6>+% zBg7OmFA)Vblzg^-+|qg!*-v3bcD(wJTFKRXg##b)uOcxgV~2C#jal+_&Ax zFSKSsdm`$)^1pZZDH$D{jKrb#LRbXE>NsTnX}O{%t}!R>;Er5Tu}0!FR; z*+!ig>Pc|K9OH+uQ8`_v{P2+jW*)Ca0Hc!cQ%aU}N%59e@QC|-F5f)Y zz3J^r8(#!!PMjbK3V%;apOl7(@2T3{9OmShQEbPWCK9%B(U8cFKqn?$ z>PI@n6;>2vY`Qr>C&j+fiA5n}0(y$126HBn*bSf8v^jmvk=J_+DnAX~yA8R%nfnJ7 zwS@-B&c2J>MA7Ziyi*=FE^z?F>L%K)=k%wUN}Eo)ZZ)k-Dz+Be>4SSpf_p|(+D8yht3c|tN;2%UDxpq zQRxv#*RyN=jc2*-uOmeGH{bJ7&qr49k{lD%ANBk;&ihUT**ERXHH&$W z)M-XtvBE~(iD!$`h8dBOlGf9ZP}dzE9zIZ?rMlKH5fBxo;lak@-svVK_Pn|BH~PM( zY}_MR>GOG8Q4|yICCFSa^*(xt_{-$$lh;qp>L|JcvBG=v2D+URR&syUjl4k?GscUB zUdySDZLV@n3lu!YW~zZ>y{1peddWVV`l&WrUy6^QAZjY}2^j@&%44z&bQ%W^FzYD%*agF4v{6z_lCgf zmGj5{J&0+YkiXRW+jPC(!HopviOk*==AmRaqj~!FCiwn9T)*eLJcqRSVfEu@iG44H zi_78Zj;AuiTfrJ%wyvWZwQRW&z9OD6<1j+gS{P-bcaAv0sfFwG{-2}dhj8vcC|lzy zQGf3pNSk^(mlHxT2t?axSSr?qF$+xe>l_yTX|dTVrf2-rd*pyG#<;L)l{~I&{xm{s zms2XDN}SssJDAL%u>w|ZWu%$(Fr%P`uTo6)79PD*CAa5#=^ikDG5AK@pF?>AFjq@~ zw;*3*F)hrufLZuHd1j8C!vDP@oK``G5ZlutlM=o!iJ$pe_>!2w3cq^hQ{wNP{{cSu z&9!&IIuQupYZwsMst7ozK=IY&OjaRDpi}SZIbi2`Qnf^gT;>Q;2l-?;01?r(qnr76CxKw{h*^1y^3 z+skf<5ix{>e^bpy(^Vv&&!Z1_*nKriRymrk*)yL6mLCSaS+j@-E ziSn-~TIuSHJ4ug^Oy`hN+i+f&W5_|9sm9UPFjZrGc^FU^5dXB?wxuxNS|;&xr?z2` z_wsmjC(NPsFY3|%L)KeHwY7F_qfo547BBAZPH|eaMS?rQ-JK%EJ;B{wgB5pzy9M_a zEv{|%m;Jo&`ObLG`IC(NTUqxSbIohcONzv#R%c3h9`MGPUy3t6-qflq@O`&dnZQZt z1p(706CFVZ7NCze2)tFKh4je=kA;eC_#vwq9lkOsZ#nJb$`ZM`dG`>i`|5E6lOIMT z%QIc@YUG3RwwDzggupNWOF_k2fR$rg`pQ2DmghB2tWN(%XZ6pA!zq*UNftc}w$!$K zzQAAB*S`}aLeZ&q*+nn;KrqyF3WPG-k}rH9J4{BoA>;;tF0N&ZB+%>9s5CrVNT*U$ zO#Ugt^OB`<&}X!bv^D$_-&fyaSe@>h7=!5vXD%u=)a%HaTFE2A8k;Jb6Uc;+h6rqNbFoJ`J=^pQwUg|}5zv9!D#3#~F4xuW@rvZQ3$pEtppWJ9?lQn9Nkx4t%bY{sU z9INZL1YJSVA~Fj2`r?bdXAZJ?p$pWw-l9Vs{CQ4A9De*xl3+%*u!CA~3Q^c_DKU|o z?EF(%f0I$J;2W%fi)p=bK51hRE3hsS;0n52Gt(wXkH9ytFY9ZAtldLNC#nr4o>sGhC zxcDh8o~&h(TKsJnhcmJ5cMlUu8QzxWiW@W32O8xdUQ~T;6Z4GK*$NWN2zvRo96Z6( z^IhED%xA*VQB5F4;m~Y*4K)B&2nJ-^{U!vk)K!<;GzSTzi>y?3nCZE3W6C( z{}QGRfAzjz8)=?3^uE@>Id$;xp)IfFI_LjUTvmi4^xoc96g*Z~_2Cfx{dD8zmC?|n zvXrNzqT2YEHc_x&Jv@hyCw82yC8Z7wNNs9#Y&=#Ad~W5+%1`B0BUab}E4M9q=)55< z92OY}hdq3y#ONcx#y(ja2rC1}oznm(2m%%R_dGP~!eg7x%8b?pw0w9o67k4Kmu;70 zx!!jg3so^%%;;)num?tZtuCOM{xMTICL`2ux=Ra;;Uf(Z{E_uR4S1!|+0y;;ol$gmFr1jrVD1iHf#!ZRd4=!gR^SZCw!Ogz}zb?na#$<<{<_B{qV598U+;GX)8Sl(KE&D%je zAMYlkE*2ozPO)&_Fmr~!Mu|aPLth6WlPGOC^xLBuB>E*prUtCAH-y=QN=s7yw$b7T zYwyoxfb;q*We0#r=gf!t1EbS~sn??cNEB=TQ-u@!p35&NPZFw6eATi_->_;aYg*2FT|zk+p(w5AwEk>6))E`%v6YXP4Fq;txmwHtk~H~HObtY_fs z2V15dI;8Dj&)dIu=k0rg1>;l(T8U_)=9KjH>+JO**Vdfq_=Dfk-VPP?8v7*aA-zFB zU}DcmRkH@kvI#ork`%)3!VK9(PA5zaP4I({7zpdM5`G1mFBHI@$Hq*mS<`#MIpfz)YbZHfPMXWo2hj-ig5yyQ0%OBA?nLe&Us}<>Yp| z>l|9ybyP|5gCt$G6<1QWIhvIAp(zVA9}ve!PtNt*-Ab|C2sz$N>l{XG<-Ur>Q#gUf z_5FY&vd{AlKmFYQAS=sPovltBe;t{nI#uxxmss7;h1bfo)3RLGvKV%E&#pD2VsDPR zm>ok0uk`9NxIG$CXNFx)zdOh4`;$I}SG7KB(0Y8Z(-f3!?IhlY?qKLawql3>GneRKYZfF5q#zxkQ~L3;{dTCu(U*VdYx z9*}YXW|dE)=DK`c-T2PW2jLBsx;m#iyMGPYIB1m}5)&Pgic>=!R$3f3ap4Y@ANRhn zXb0g&{e$oe;DcHBwH$8tX-Wmk|9aBUSup{JZk`OS5@cK)(TGnautu-A@!_y2)6hKVYA-w50$%O>sRO2 zB%db#EEiqAJ8|Iy@%Yr(K1tG^BTb;I6=6#Lxvk2&ut{Q4{8|U5j5gy7IL9`C(;{}Q0P;XwSr%OapA|cCbuNM0 z&T3RR%aC>+Za;P_pgFEMSY-;isfwrJ<)xR)jgnReiCH0hoBUO+Ooc*4@or=x^QY%} za*ux>GR;Q1S?u<2F{e>L+92#9O5Gshz?5twa^bSO*^=WQgk}vrzi?g}Q8r`d6p6TI z1?!l!%>F?&8rRW8D~ok@f9*iNn-Mv5cli?U1Wv~yAwM6s>YD1}NtWVCjh{gz z>guxP9t9%JmzCG0-j@~lG6F70hgU~%|WWmHg!#UmFtw%-(G^=*J(s~#v z@XJoN3>`jOS>06>zp|gSod%LcpVJ3F*L?tsl7KF6ab8jH;LWfg%AoAuX=Ws(Br8;R?3m2 zD_8GqIR~oW6ZHtVmG|l;WSDxbFj_tNvz4usIXM_G!wZprhM^Oy@{ZT)N*Y{*-`cY4 ze4o~g({bXA;*orl3PB1L{DnU{fch!+XFDQR^eD7&3d=;8^6k5_?Z}_PY9YRd_9DDq z^$J2$$Vr{Ui^^-WkE8Ed{}g2QJ+i0&gYYr8W;R)+-QIWW&ZIceh&A9RPNvDj>x$;< z){b|qsGHXeT>8J+vntZIeA!{JWl*EcYVa!N?5NC#$xZPMc{iQRqT9tr>E2muZI73T z@Sq^TW$|K0`QJO|t;j5|)%jZA#KsyQWg0{XX{~G1gcBR9`_>lnCw`sQoHPtgHeb+ZOTXvj) z-TKzrZDm=(engsF+x9d(Q9Lk=VZ6~|OMF5~rg~V{!xHgCFs)Yvzo_~Bs8>dXpLRMP z?0xU(cPM)#DN9GC+zR9w(X4j5$fHRot5%Vp#+vbU<-?K;FN3iVyX~swILpK+L)3SV z=X2bxK7gTM+CHfi+GRRdu9ec+a-^Jv5G3cL@h4@5d7t!HN?Cm(x`{KUd6D{y6jU2$4| zfkn2CUS4V$Jbc$fy{Ecb^!6=VY%E<}mJfxoMU1Jty}r+Mvdd|0@5I8H!y`OxD71SP zKJfm*`F-@|Al<8(cc5+8kte-_C_LLO?XsYLJ{~Af@5FL}T$y+C#@Bk0pNjdX!cr}VvrIo!RU#D3OctCpm>0E|@^>l(AJ0_!`(FCpy= zZ9|p<>B^@Y20B&MX^%{0X$?)9ll72OosAkNcdlJW{rQ>hd!%Td2i25&8D7M6X{t;; zdKTa#^sSP+G2Fe|`KPM<**Mt}#P{^Co>!BJ2JZ0o{g52ZpV=0O*M@(Q;>_jfmr zwc??Ib(rUB4fD?uVrNKgp5{d1i9jw~-_BB2Rhzb4_AtFNaQ|4jAy}2@nZ&YJ33)k9 zb=CG~#Y$>&M*Zzr>Q$UT2U2yEDHb%LXOEp%Ai}dt-~t#@;!&B?6-|+oe{A;M1vLCu zYo)p!>6KUU$7PSumqg#Kny%R6&9`_UKNw|WvK+<pe9sGf4hF2 zw2JbyyDL`arWcvL-$RmCqkgy!1(r%1e&;c)-;l@;9V2eha8uU*n0g~9)FxSUhzOwI z1Obv|Dw_{s0-YP{g$HZ&^5-d+X|3GQ4;0Os=zRvy@xeiBPc4i?ZJakm^&c6M2TB?! zwf$m*r11H}hgeb=7u;;YiecqnLIx6lZmB>88R}+k27BS}(tD!bBO*A~fCG@a$DN@2 zx>eJlRk-bv*Q!qIzb~EtppH5};lIKvY#HAEWcJFHc{Q`{Z`d^_FCnpIMyz0>YGuv! zMOX!ZQ2hnwQ@*}s)$T7Tw9tah)P?r*W4-gF`&i(QCej-NZuTUET-&Kd!by&hcX^nW zmfDCq0q%6@^i1?*W-DRR8SWq}gKs@FHQk;ZW;)>_ZDjjWi^F2ZBzGCyFZTFl9=fJ8(+%yp09Qz8Q7f8&K2vN-21}rl=HOf+Ix0CP}%jT zq2#{A{Mz-$)4E8ZRiR+i+6+DIC`OnT=6n1 zoX#8F5p!krqwMkBV~Q@Zn8{^#nl!b!6v0PYby4gI@`@5oKm214UGUMT)1xK* z2Vr_e6AY68DIzY^7|9z6o(@LB-b??{RvfAkRQ%Df(IS&)%rNojl&A9eKM^*wG_zOm zOn}CG%}~c^5&1ZzPeshhlk|MDwFWX5-vlhG=pz__<095egm|3#o8BOUKFWQbU3)%M z`OzNot=F%nhh%)F)Q zBfYbCku985;ViY~qVJMNo33@SjH%6X@SMUXOD$uK4y&MZE#!(2w4Ij^eUId%50JSXI9ZQ{%{CkQY<85GThL)zL{53=agZn;8Hfg-;;6T6Z@Jl zHgV{EYc@YXL27k?r2qN-epP8dM^I{VjiHGs+b1G^N|k}g)ZA7>r7^v7rRM4$H(eJ+ z0oT)4c;jSjWA`-<(`jskp%u+6WtM`RZ1L8&naWDPw0{lFomb8NzQfI|TyLKSYpU{H zm65u{xGY!R+BHUx1@B^Hx=G^KoTciT%WZhos^%#321{)K-t`{Ala8$XEW9<_ci^p- zzEvB{iQQD8&CsdkPBXRagA+DDTR2HgTLFNtnFDC>o|kXv5I?ccvZu?2GjrHgs#F%? zqawH;fLDgGQ;^U~OQj>)+AsBW{&jo3=ar>{MVzv=iAS}47(EpwepvqJle z+Si)$$L<59HRnM#0-#x`kG3*Wt5|J(p;KKHxMg*^4j+VSsGKW_yJ)7rve&7JJHQWf zAyo5KC}A(8+8uS55TzCtH)f%!marSdGLr6#!Lpuz%-yjH3dGM-B;J!nV-ksdt! z<$B6w>q)`1CP&dfs*=;H zHqP_rM&gQ6;2F=S#?);1260ln(mk!oFtWFrn^5_l0FsMg@;cGG_z!|j-kaZSj#9UJ zfAeGi%&L|A_UWV@|0CNn^NcBVG)5&o_1ppH9F+Rtb|e#x@@C2_y+Q`JU^H1nMSLX$ z#ltb;RhV2!ad?~{>X$6Vt79lvPFC8z%V#OTP37 z!y8k#sV#}t0KzRSyES=3174;fD=gudT~Y&V336sUHHNjNtQh%cxpWFz7D+q~ma4ls zX@151%x({jgT?B{@|7PL%l?4E&%muHWHz3;Kc^e(Qg|{cih_Pk9iAo15>!XRPSron zl{^N%8EOD)tIt2Rc#f=_jCZw7wrj}OV~>6Rf#n6xy+NJ2|5d($l$PJb2y*}1eIR@UCw!~K&RJ}0oEC!_D4PS9O^YR5YGZM&x-FeR!!fJ`=${hr2^Q>WF5mN5);*ytCvge zH998@OA>>#vZ18(f`=ko1{<`Y`>HAYNFIlhc4_6sR|m`}%Lgvj1+}du$1H$BsxB#;1wq*dOslxCsLO`s<0Sju@Qesqt?R zv;tDARTW~*mU9#z^gv3Q=_{?6J$GsQ)6L}AJ$RfW>*v41RfPuS4wJ6qxlOTWtsKUW z7T=~OqADoCBobnqCd@5EEkT;dRZk7yc+I^dg#VYzb<~Ejr3WGBW=gjAK52* z4eU8hoA29OeNrFD&R1|COP4OvFmfeZLZ}xV$s&&|ffmneI4dQk2oZdueQ4NhWpVXe zIXL+8_247S`wn~oMlj2onX%?*jeWpp{Vhi;e@i8saYU@`Kse&@I}*9Y`Z)`7Hi`og zC$o=D8fJ-Ti0BR;3->WeSB5P82+i`u987A`_0TJEENFT$LVM3mLzKI_L|~J11lCA# z$fuWcp=+u_?UO^y{Sflf;9FF7lyW_jufedDW*gzGldZI$>&uS6f9@2EC_y7n#Tzoz zv}=J<@8qXDSmZW8-!zN9J>8|4k?c8V&)(BfqgYq(;h!ec$*Qw-@#m2Z?HIB1l^4!I zs^d}fo&XuiZkX{IZ8|G}2hgPw)Hl(f7RBxe7Kp0y;@0*;J($66Q~bfc1!iAu)DMFBvl^aVCZ9c_u;87kZuiFqYi1ayV?<6%TNf?%kRPfm zOo=G8-_fo?Z<0l}AwR|c^oNILQIfh+Qx&M@wN##;_7k!DpBJ1MNz6jGDG_XMz}mNv z0n6dX8W%1GNAUhImt~Wmqa1sFif0WJV9O%k8eNoN-#Gz5JRswzn&vj!%F(%4RAc8v z+P7P4J2^BNiE}W~UCS-W;uTUECL9kfgoQaUkAFk?bbTx296}JAlaaxpE)HN(ShPqW z{oWeNzy+ykz5k|ZGIvI`9Qdhi(XPEh!f~a_5w)u#u?@q_hWkBYqdP|mZnoTwz|uTl zv&dY*(huyzFYct8xB&TFvkhoSBF$2EQ*lv#9Tmio<D~0G?>+>x6kZZkN{_d0y=FqdOM;x7bVdjE?tB4pE0SOJkTua0sXHV7N$#wOU zgu&PHf6NipP~xenGMU@C#v6%Ep|Duyj#n}UNyHQZkr`322i3=Mkr&ytw#uatvOqdR z76uGyfX9IHc0`Bgb3-JfP zIjM^y$*8{$@BTsHo@g(g&xdC(8`X{?nBqxFDu zp|sZ%Pzx9o>#2}cAjs+iamr7!8K%O-iMRBACiChvo2{Eqonoi}r; zkEJyOZabz~k=lqg>u(Oq&+m?=!s82f>A(0X>3v5Wq;q|yY2vJo9!NO7IQq#U9p07h zdB1*Ibw4;h~Z4tb-GP3t51Sy@lCLrzu;3* z45K@i4|B>PYw(xp%pfU9WC486yR~jt83OCk#s-wmiP(V$>(8uTZLqnSCMU9P=UxzTl z1UPkszcDAD8pl0g#8%bj-(4~{E2+F|hMkYv>dac_E%okLgPIK&G}mQSt2CnUc)wOmU6Lt+woo^#yIp4lvfK~(g%Xo_~=hb;v^Z0^DGp??qrlTLm%Y~HQULZKz1zck?Hcxf{o z_Vmfk>Qmoi(9^Jn&&Yz}_L9%t9Jk(rG;Tppxr#c$$_CAgrKK)o(C|2$&+c#Hi4Qg_ zqGt8E3pD-qheESyHr8H-@EJ^K&o3Ty0z7maxH6jC7e8CMG(&c*T*~va;i&=@6`?uA z7FWgF$61Q5k$vasWd(x|V^pRIvDWCpX3>ZD7L&&ILmO%Xay*nPY@ydHRjxl8WIh?; z@Ovne$%KgEJ-&7E_QKuwX%+M7A@#9uYyWWe`aKbu-qRPgGre8#7v=J~rCv%%81 zLr1wpMTF>LaQo-+9^$HI?j`$>@&rYLYdMapuFL?d^CY1o^iloP%le6MNr_~MWIXgF z4;>ZgoNK#iU7IleyGR8q`p}PqShm?%dVJO^8Oz9=#EkvE3p)O>*vZ7ICt$CT3~Szt}CaGey>|;k~hW~kgoFG zzER(__bZuk)On6u?ZzMN8B%)@3qPF1QAjtnnaXtWfow_$5P&o79xgS zW=Dg~K*{Mk@-d)L1ePB4XFPsz(_;_}`Jsu&S++q@Vz!@%6(_4|FD^8znT6o{GW$h! zTv{W{;1Tyd9gl?9jBz~jUqce0@hMA^7ac};+_=yyE?i9QV^0CQi!!>-haSaC#<-@7 zLUD@mH$}zLEK#Ua$G_Na4r*E$D3hDXplhH+bHwp(pNZ&ay2U3u^X33H2|Z=oHD;!| z-o<{Bo{;@zvbkm<1T?=E2CX?(`+Ajz_eh6R6ogB%GGR@!8w)GfN03oUYY?#p$a3Tf z=mUwcrNp&=_HJ#T9vFHuV;o}^-CS@dTJY-80a95N=)qRJI4tE9W>C(aSjBA+1u%@M zB&q>ts3dliOF^5eh=N9F1gT2nm!IzX*Yn##(+D+q8KeG>{p!4&9OO6DT);#1_EN6f zz5_|bKC$cQc_K7R+cttplpAm8800X5Q$54h?{u6X*%8O~-K5T7nSl+`yLx_kdWlTz zNwy?kQ0b%t_c#KBWq{oSnKo62s;Mn>tB(h^^N535iKs|u!Jd@f>UP-{cTB+hIIVKSaJ7>Xw}X1bP0`^gdSfkC7i8B_Qyn@X+k zlZa!uzqpy|y6%C1@7!nN){BM2)ldLR!1E6FLHVa+@?QZ|=Pcn+m(5@5yUUZS>|Ji! zLZ5k?WotTgj?$_*$)>*8F0)kyZLe%qzin6Ck+m(&_TI4KEZQeovpzhU-k)3_%JN2H zv=01CyL#enRzlguopR{0Ykid!EEJF9MFv*ENaLZjye7CoC&d8gjEcL?1q>Uudgip6 zL=`<~?c=t0p?4W#H?G$;X72^_tYKyGPuCCGJTs|Af2Q-OEn*Yo%~hnRNW2=AOUjLD z{GdM>T~tOL#cdlEjl@MeIhQdGG?A$aCZlK|T058Hf(0ck&+F+>3t)3*vxYPJ8JMP0 z3&(e=ZXnEuYHhwlj-}}+j?aqo239R=ArDLzk41dgE0)>*c>=$uEK0|med2B*knQTT zbc`9oP42|Qr?EB*FlX#x1&tQc%>(4%>iUErbM$S3WMH6g`r~#8^7}$bo6B_nKfKFS1})o>zrw?KhNz*7Kw{@qES}<+&OpGhbP-YKA^RaZ=p<1nRFW8?YGsD1qn7Ak|WJ8Ii3Qf7* z=v0h$E2Zhl)8@&3GIn;F6V*A9Z6Bl1@Iy0A{p5>`=g6=mYBXTbCdu?)I zVpCTl0#xj2bX+&_@l&Tj3}tT>M_{uxn3eVoa|6P4xcBL6?8!!Cf=SsK7Ti-SSFcCz zp|4FH>Y+Gx9G^zB*%KFspE5i9Laztq1v)ytNlL=I+%+E@+0!5s>6cN6#}kZWsFkUk zr80^*#FU`Z-&nnrv%eHg{;0ZD>5Ywf{;DJ+zQOS$W#6v=*qGl3)KfQ~Agryz6;HfX z)QXa^Qd&c`p=Fx`MugL~aFT;COZWz+ac#b~cMvscFq9KcCN*JCkK|k=d1c@^QmGAz zdyZs?j+<`I5y=!}KzK;V)5j#I1V3 z=-1p}4u1$fiJq$2zGB0h@5OPAEFxSd8QdtA|WlGWG7rDuRi?L z<8VA0E-*%j`V;^ZIDFtxmR4Rv3*%NFW^_g75L_B~ zor6qHz`|B5fW{Ddg(RZ!>}9sA+BZYuQs>qv+;P+Wzqr=eoM~zB8S%2Bb$`m6UOU%GcGuEC{>+S~!F5S}BdscDC3V8#xb&Fh_I>-(Gsur5^)v(_K%VW6%x68SWReODyW3LKXvs4t7vps=>Ur z2bD0sZbI%6;g3KD=3PG2icIK@y7f32IkNwfoj>;*6?4sT85&Ilh0OZHfQQ5!gFOMG z&7YP{BY4&_b!)#z+neq0E4gya-+!F~a|jYm)j?WzXfj4CROFaZ-&$IbKIZ~02=!hc z&=e6NCw9!h$VAWe2+G3IRR@;ociAGCeI9Y*gp1Na;U^~jc1w}=x|EI6^Gt|2GjiD0 zcAIAB-d|8%NLaq>PPEC-r2?Bmi9V1C@0+|s@xzD9ylIY_BK771uRrwvoyi;hZNqPR zvy<_(@^}7G^z%ETk*)^5pfc~T1cLmrv@pH?$OxQL&T?Rgg*nK5MdD#4@_4X>pnZm% znqh0CCf8P^NKsx}X~KCBSMtczZl`%B6rOpn9MO8u;r=}do zc_}c*X*fu84oKWGf9uGM`Y%Y6b(`1cAomT{Vgi3QYh-0buOj21;Yr?&fM$j#BEfT$ z7Wvw5+TshJp}+s1Yqq{MmhyL+jPdn-!2h1r?&vM9r!4suZ$W_#l@y z3L&y&G@tUZPl3gkX7L{!j9Pj`rDrFYT=d6{qvL#Wa7$vdlB0dDyS)0bSzFiU2jw{Z zJA5WSrI2=moWm6BkPm!Tq)iDW%gg@C2>Lk22VGPAO3uKe0a8zSYu9AWRe)6!Kpkm& z(HYnE4}w3%-eQMdhEt=p@j7<&innhP!6E7B7p6rdrJH!Txux(Mp%0(QlXz{CUE|*f z2Y{eo zUso|eu3Nq^c-XyHe~e6wAeb?Qsjyg6h{58Pj8?`+v~nNRk8}EL6FMW6OydM<$Cr>=#Y5KVH@9&M!hCY`+K3Xkm#N*ld z3tx@wm<41dg+eEC`Ae=o%@pCJvr;~A9^ORUR9$^sd2@l5j25q^nUQaKG90MR4Y?6tW* zOPv0z))x=W`O2JkTh!!XM9y}51YzsA8am^xg;KMG=eXx5n=Ydsg8v|JK9RdVU3=Q3 z_=iRO7{~FR{L&qj{eVCGNwoiamd7_VKK*8XFiQ8iRf5~ z;=MBAjU_PbuVzyR{yvcwHpQd7vfWqD(>Bb&J-6+2Tu~?q=(=M0ywn#HxoWR?w&|iagtf#!2g+cY~k6ciw2t4t|NeTlSP4hP=x>F)2uOo^bPu5#BQrr1GV4x zC9PMU{Po|t_x~G{@ZY~^CdYmlczjs9XKeD#K^!sxV#Q{0*ho4MkQNJNaNxfG+&{%I z9_d)OviS)jJtCA2ajHGO?jQVkU+7e^*41PhDv|L!>*%DL?{3HH zYB=20bl{rSWh~TrRZ&<^Pt10v5@8r zYTt443xumd;SOBcrWw{r7mhVepV~%cU7h8|MB-@oSSYEan4$Av!mswJ{i%|H-(-re z%jFBc84IZLx=!41&MCZ*>;zbGKXg@1RL3mz42fx*$r#i%smEl}jD0y?9wNiX5DAu< zP$Me>Y(h00liEe#0`1npaT+-B&c4dRd&^7M&e1<1*JysEovt`~$15LZwN$oT$3Am?kcgJ-$RKen*RMZ_ zNGe03sXt0H?)r3VgQID^H83u~F{i=8brDb}Q_a@rza?l?=T${(waQku1X*9rOg`j9 z-ikLBO+ToLthRXoWvYgh0$ayt3AXb9)OWdpS9JGxeIas}PfR@gx8 zhFg0IX&Zs~pc$>pL##>uolcEd)nQuiGrlKKUyYD;}P zb9mGh7%j5tA`Fg}OBx4e>g zvtKcT{y_js%!!u;ceib#CG`G2`&~gq6m^U}(Pee4$A->n{Py`)nvbi~;@k$qJt&4f z*@UG#^5a{c5~hz@wzr!RiiG+^DKrV9-Z&zp8rcg8BT1wj2&2{KTC)A)eK;Bwr@YCV z{#E#iX~6rwTEQf)<*#<`VO6glR_QXPmgn;T`OCJKb*|k*# zEZGfDbi68ZFU>FB%6xE!PfoQpHwZLO{*?K-J5p7-;CQTXnyz#USztCS48kp7nP)#u z$4pvfagiEjR(13{@m|U&prN7O-+@TpQT>Ck<#9+BulYrP*Q-=$sQ7nYxCG1_wAqDF zK5n0n7!;PC-0A>6AH?RP$%QO%T`rS%aU$vXrD4nM6rfg*Ww}r#(})C?31N(;g5@qb zfsHJ-zo6_+hZNgW%@uRLeD9+g4}F-AyHE5PzL|5R`AIEx!dZA-ME!81oSHR}oPuGPag@qYO&m9s@K)8f*Z>>X7GO;>ZlT>=W+*?)IW0r+Y0vb{S2o{fd|JY8+&P6&B zGfaS*!6L}T_!Nj}qkb5I-07tBG^rS*oYXo5oDO6y_i=g2!Oc2q_!@wdb4U7PrP5cG zlt*!+2Osi1{cWY8hWl9Vuy@IpKd+T?=APMT)A$pmX(+Em&SX)B1%pTmQ2mLT+tP8E zn&njI-9FM2ssDqZHiyeAnLHyN6y_Q{?LdrY7k8^J8@KN;ZB~9e;IX#ZKubkI9F^b7 z`d2ySV;nS#nrQ~kGhE|I>cH%v;|Oa&W1wscrNm;?TspT>}u(vs_bk5dM05*p+U)Rl!F z-0}pr+Z**|>6=~Y4s>ug6rbV^I{WN7^4j|n_Fciy!D-dl3et7( zqc9+a`65cQ{BG5#fh=;5GFF`9Fh=!;kS;6?2^-PkB44bC_0zml0iN;3Lp(Ib(*M_X zhsh*?V#|ZBmJ!^D`0KNo?dUq;1(#aE0VxH;%(aly=+iVQQ9IQL_Dr_u*M!}a%J^=L zZx-iD9SC3Sv^8}`tw0U|1GAZZc7SA)R!FPHH1+1V<2NiFscLZ|>ATFa?@x-AM=BcdSlgY*lHhv2n6Pf*)E=iF+K3)AnemBj6`O0mTd-r?p1-SdI%Kn#aH@yGA$@QPFX59umVYs> zaX3d929GN_f-MtMwwWZsLhU;cnIKsTPg|KoB2#{X#h(sEx#P?qI78+)>8j%`oe$p( zT13chuy$Bp>K`dl?@H<8`$sHKl|hVp*$loemo$sAS4{>N~0Pc@w%Z+sT}(f zJ>q`J)y|6(Dt2tMF?;2Q27`2)8z$@IuL zX$~#-p$z{UE$$n%aa=d)9uFlE7|C+MSFeNmNqE|MUeM z_zBU`RnrYhLnIFL_UEKqlj566TwBobvcFNyi?~~W4EBNo8lta>XihN4t<5HbI|r{S ztlD7dBf*07hs|$`#BCF_?ws=g;$ka6WJ1$czefw;(Ql9H%KCbQz?KPOBNtA6$GkS0kladA<%$;7BcGiJijm~ zi&6*AsTp-8k4>rIDYJ=ji(-=8h=>OakO^DAOmCj!Mur3~TVbU|sE0?4{=8^@hn=ag zyPuBje6$}U+`3VURu2|P)m-oJa_0osX&X%F#8UlN-9BpZJm|3v9`rl_i>u}D`Oy$$ zU(o^4`1gN?&&MXE0LKu%H)l**5|qcW9hDMmmyU`(*I5C&f6GD@!)kyFO~-ubW>#E} zYC?A&P|I!mnGw2+xHMR&a;71DjeKsK&K=3;4g7>kQ2u}+S39gM=mahZ(J}RcOEjEa za3$&2Gr1m)vD3yxl!FqRkZp{QMN&4CXrLH_t6wFur`%FgrClu;9HB&ToBLvg_o;y+ z)$vgpis)QD@BA!0Z!>xw^TM~Uw+>nz)N+zKCLVd6?{u7euTg#2gAC~|*2yAbadJx| z9GiStAl9JM4gacE7po=3@#mjtW2=nbUHN?0jpt05;g;RGm{j#}z@NQQcmmixKHr1TjDrM1R zLYV{X)htRF+p)JhW{Y&{NP@+LFx+R_gu7gy?b?ZblBCHc`F+Wir343MxZ?4%!q|ZFXH=+4;}x77ccR@ayYeZ8jVG9Z<64 zT7HsI{Q_(peBqWEuJ5hiM->o~lUpc+GH}zCO8JWTuB|ldc~V38D<7H+0|S$&9jYbPgPY_kqMJ$nm3Y>0}dE&Ye6!4 zVuIhgM0_AiSjaMSD6)cKpx8AKka0lxZV-pse{$mOX#VH0ZZ^vIN$50CFr8x0CV^XoXTD+2Z^M(}aA!1gVC6by*Viz`yGe%mv`DIq$ zN9Ux=D=3|A`>g&l5PUq};V@fvx{s}~!kgK4g26dC2N3pZzD*k(?qG$=B9djj4ylh&HlBlh;cFTy$?WSv!){%p5of%1N4cQ^g_3R$AqA4QtABi*nO)Dj4RhT2i+W}~|_y-y5XcHR4(pU%f4 zqB7Jgp=|{gmm|T*H>Ni>*)={S+u#qKEpkR}NjUo?WH?^m9qG!hLX0x(EsHg1BKHdU z#2=qEat$=>q8KR@aau@cqzFW66h3q94KAcYW0mN=l>R^E@6G5o{d!gYy-%Ip$n&rF zIioJOkx(Um24v(s%*b)0gU7W+64}vAKE&xNFhUjG+z>m2O25lyBCdvjgx2CSK?Nu8 znDfsctpTr|i-6nfO*?CjaXom*=TH_9A%jFbTq-9rQQ@dnDsk0UHRW!r}sV{rm3LX1x^BLU0kgEE)JU{8zGo98+0D_Mw|7pj&^0 z+xaQ=-S7`Kis-HnRMqABm2wyQi7E4G*Y`;y)#@;w9NK4;#{!m1B{__2^lET!?zG*oNEaOQ9YA-OHOQe}=3;-Z*H{en6q zrw=1rvA_Qhb#K`nH<+z$nqoU-k1Ll)TS}H9q>Cx7+x9)x2Uyu5Z=Gf3z>e_yg{slvyo~wj2I;iE4q$hS@5WZzS zjn$ltme@L}@kxt7(#k1IE_~6l3{_1YB4ct8%1M?^z`9oF^Nse6obw-9%UJ6x>$^vC z{@`%_lLvexWNqOm8r3mfgaCOF<0kU)Bjpb3^4P!Y;9OB`7?&ZDCM>z`Z;nU7EI-V1 zX2=z#U#nN6!O~Yt3wmR3e>Q*opL$O9+d{r>$&d-qJqOQjW#SrIW^k#-CIZhU=pAlF zI57|{T#z$`{Dl+#h;pDGEU}Hyv`e_rQVFX9f!)k=zy<|+LPO0}j;&?^FCLo7FUxJW zOEbsMy#*mh%Y_7f&Vny55iBFZhf17 zw>$}~O0zP^sIl+nw4nR7QUMib!)Qw6>cMOL9!0{!GPam0Yl|;RIMY8x@nurV3_{|o zbq)(8z8aK+%O$ejPKd+=b=AsZj^J(o?9z0O)MEj;JyN?JCfndrZd&+`jjjWDN219* zRJp_8(RjJyOb6nc%O2T`6b{OCs9iJqJs85WaYz?;>IV6$u;OQ2cnLcey$0CPzte7H`a{v34~jHNCGR=JNuak~x*~3`WHU zLb-)`nUr}=3T*Y{-Ja(2&Oh>y1hEJtKEQu-KpbX%zMaYc`J9a)iit1r4}AXr=MGsr zZaOE3nJVMVvMlTEp1S1_UYlI^aiw8ni{UaQ$OJVPk>dbYLcXwul3)zi^BiOHy?x?| zvC!s+sl!pVp2IFSs@RE=C9X4WSE;oQdVN`%ptV1NWGhD@*$7Ha4lwB0!o|^To+FoP z$&TxTaQfNb4eTDbL4Bf6xgdEddfn+KNB&r^76|Jq!8<-L`4<1=~gnCzTGfKg><;yJV+wju%2r?O__&0monxeU>9v!9Xb4;?KK_8~U=ej<0~O^< zE;a;Pk87O6Gtz zktJ`0=;|(TEjB$0mz2=$hn3ujPLse0kirJ43#%q@Ib5!$&BpF=(>Mu8T)Seh!B#fF| zz^F-HVX@7Xnt-bMP{}n=RouVgFId49%uT>?$fv>e2a&=c`u>61E&YB%?@qtJ3jEy^ z-w7g9s`XW$26IsyqcY;_+NY67a!ONo&w>1<^Zl32oY|}9CMXapm~E6fT0rs$YP$tC)hD<6vB%0XkxSZL>KHs2`HpWyS5wbpjh66IpmsM;*@FX`05AnDle0Fw>NNu@2@#Ft4H>r z*r&&>6L0fvCtvpV6g{nBE1>P!(l;vSix`pttQ~AKtAFKfCD|Q<9_6gNDEK64gk4-B zl}i&!;C)3X4y$ubO{reuYYx{@4c%dE>pmb{G^mCMD%)5OCs#{Jp@~;-l?B9UkUfOY z?9Bg~XFB8S_s;Rs{fvtyr36nYtM^@MmMGA!r>7Du=Es`DWGhBohA81w=~84qPM3@ zeS&OW7lx(mj^O#67wqjf+8ggYqULC4o`05wPvA{H`WUj@3Lip$qu(_>^g8{g z@2PQpZUe*+?lgM{Xf===df=Yt`JU;^hHdMQ=@a4jx*%J#Nj$bP2GDKK-|ig~I&tT1 zYFuS38{Geu%mMKpnXcC-<~P zn74FJn2B~|6hYrqg{O{`Eh-0-DK4uVI0vWAj2aK^q{HU$1_%$WZH3N{?T0xq*{paB z9YC(P()Fw152(qL0J;1u>uXV#Ns6QebPOg7wZH3a6BA)L`g>bI zxWniS{|lBm@}xWbsaMYAC+ZUplS?iPkJ`-e?$*9y+Aiimntt7KRTZo~6@4p4;MC7Y z4Q4SD^Pyxb)f&@axyo$Uh6=F`ykgO_Q|u>&TSI0XK%Ef!RNxWtm>1A7fQYvhNDA9E zi$=qD{tl~?A)hE-DB?aMLF6qwa)m;xG?Zav%=nL|DCx9syX;&;%PG?~#U4;Ut@i>} zHF&_{2sSBuGOA)3JS7MSl^2LM=f7Yj#!eX(7k&;KAAR3-W4*f8SiOIBWnXMHv=(~( z1=Ddy0^7%{Rigw%atH-aYF3R0o?ffBs^6gKeZy}cADD^iCVas{)TLa%*SV4={kVbw zO-&4&nMTQM|AHwHxx9MY^jBwuaxSTW#2HjuelHR#uYSlc0(A*nhqWx1pnZ5Ag2q|u z@(kBbzC}$QeM^-_#nslL<-V5Ay9U~sWmkwzbpq`n@yZI+3tAk zR@aE3QyiB8>Jr_ywyLmO-F0Iqvk~eSOz}DNyWOx9+=UK!L7jMSCY(A!MzGj~%*;2F zWxJ3`&OO2*U7CQVlyrnKNm1S$uXZ|6qRgxy8@rt}AxM4*`|c~m{Clds$8EFdDoy$A zD5mtPc8%pLd0CWEm!-jE= zc;QqDl&^EU%Z@4AklskCRyiF)-^y+vNGM_(u^R{`#HVrrRsdJiQJ#0A7|BHK8MK4G zHm_1+OV~9x+{H;gM<^4pUP8R49RbFtESoXl#9Nsc_~CN*P|coU3*Ii~5CuiculwOi|#=SId%^~{DrUv^(F?$G;P zzS;j4;LIQCReP~{7CX#Qy(`KaHFLmf`GHY*GvRcP5Yt+;uhgib!qv1LDtAwE&E#hZ z=0$PSCr4@+B`+RZMeW0_OhmhN;A|xGL(}4^7KZlMr+E@1>}|BHKZF{i+^HG(@Gl9)eo0&QN;|O_ zCa2Sz5N*fAKum-r8*)#MOG`=0G!PbAL?iOMlp|Zu7>?#-4?ah~%Rb-#y37vOA1lt1u7jHnNbFLnv8$X6EYVH4STu2%dzhE zt-ySu@&C*Hq)uULo5;&dI}dpoVN1i)K4iDLYG4X^imVBagBQvX@+YHwkPb`PN}msj z|4HN~ju^p;6exAT3S708tG2LIDtJ))9r;Fd=kWB|?|=HKDfn6-dHGKAM2IA~iq+V6 zMW-b=*3p-ZeTUr%%zGfBTlvn_TRbE|s+lSKJEbNJO^&7*hoWWT)oLv(Hbl2r^G24; z2)tHNgi%6F?+qS;Rm>7^6WPRujnV@&hZ0x+^pITrp{;xt3Tf?v-hkeb`9l(B$jC{w z16rD%7OObNxk*XMeNJ~w3sXsr0pYuh;&&c4_Yxa>Ag+O@bUl9UZ)Cci^dlINAO*f7 zVOl(THt{gzz4v%xl+6h~@x1yOOjp9VuqZRhAUV2LL3H^0xLR_z_TG3KG%IEf0eJ8w+n^~sHr|FU!Sp>?Z^IGgFQ;USJ6Es zgLwUt9e(p)Z7`rl0dT;tMIyrg#~DCvIUv?PHVadnAGO+eia`md2YcczTr>h zjf_WA402A=2r~sx0w8NLEC(>i;I%0a5cPRow`%bd#GG*be&a8n4%?WXc0vA3Y#)BG z?(5TR9u->U3d%q>5FnR_C8ddPiZz44;!{)I)WWn$a|nSl9sH@q7Aqb^VXfaN2}hI4 z$#(y{;<=s!`S@3IA7`H7o-Wou!oXoXv^$~W?&^>FxgK+mPt3zQC7tY-r|Jb=kM{8s z0%jr>y3l(@bk)7~;mkr#v^67a+t%@?WW^IQ*>PAOC%SwX>@y|x(uIzm@v7i+`ruWa z$qmpY+I;YV_{TMLI7jOK`gBYCobEWb>tHBI6ZT@3@2H`Tiwbe6Am8X-dbp1BJ@AY_ zY??4FuQr`GX@(;78#e>hyIW$~cL*7l~c2fnvZ4}HY|Bpgwc!7Pfwla*}r(k?XP5hMb8Vy~T};knA5 z(*Dgf78@2>B z4}qXBmu5fn;H_y`ZbB#6F|iyIp<78<0h?!uI#agx`ruqo8<|Nyau&w~EZ*_v89NC z`G9R{oowvKl@f&&9KQnCQx06(X@Hj&=<4)1VMU%@3jpl9;E`?=_OQAyyp5-=d=d{j z3}(^0cA@}(2enX;;qnUk+gg9`FA(6YxVN6#BZiOUCYUQo%n|4Kq?e@hmRjb-yT5qcuiZWgFFJ(b8DV70|AB^3MQxfqogYbCjs;Us1sjH*h*Z}eTS?ow9cO>(m4 zjQO2B+*-Kff zPydLF9NRyG2IdX4xh0p$_cj($ownp`0cn&GC7DgG88+1{;?uh6A3z;#tVZ{auWoFY zN1YCLjWowJUUiZ_1W*t+-#+oj8*SI!k!G+{O~&QIi0iYZ zw50T2r%1=q9>X*{+?Nm47=gv<8UdhCxLdu3`Hxr`Amz5Fg*=jTz`XeDGhy9ec8JS) zp!-n1*R`7PVZUNcL&3=_q=_q?2-c{ziJNwU)?Ab{eNuk*#3FYsEb518QdDT}Ii~TZ z&c&b>w>^8$YVpd{%~dbtzoOPZ7TgCqOn=&KVRmHb0>F`7HoacfLp4<>2doZ+`-Hhzj*>$oW^x9v)+x4m*Nc*SKx&ospy zVnZp}&HHjCkGj6P$garo6Qos;RN&k#s~7wHo5YJR-$UN9IUVzeoT35y}=x*h!PTqr9uo(28!pum5 zxXz`jbt1`TvJzHs@3RtyA&~5i4!(IK9!5|g>Mn!7# z)!T4zPICvhfwg1!YRpiqiIPN6 z2O*BNw_NcU(LgL_)K!ArB`4q8fo-XcL;}1Ow#MYMk+cp<&QjiV=<|Yo7SC2Vw@jIk zxUe!Z20K!q`4d~irK7$$(M9k_&7{M&bkm$oNgn9aKbV;Wn><2*wBuXkOig(koM0MEoyuEX~R8kVdQ&*HKp=Pajsbo8xZ=Jt0ss1)D`Bb%*)l5g(ech6I-09mvN|E;x{) z(}2Y^Oy&CC&X;BwUE1N*OH|dhey4I1+)47ZWvYLZjBp9}++yw!fsdjzyrX5d^@*Dv zn8i;MooMDq15QV_l)f^E{NNfWDX!!+l6251y7E^ER_bG;z;p!!TL;WOB$R*m9kQ*oxg}g10vvtxQf!k}@W13`LdH z0jZfnIb?KHVi%T^&;{|*zNukq_AQs+@@Ye>QaaPHZBBSv;u7(RVhz`_zgb26KxMGiAd(0l5mL;8uaQ5k|Cqv9 zUD?(%%iuED{8rmiyAuyn3(BSbPod?1r}r+z2GqY247~a}a_;$7cWfS}gM9x{t~mKT zahSe0@I--}ns^pcPFst+t1r&aHu4oct6#1tccT9DDsb$F@#k}yTVhsUW#BK^dE~BO zcQ5fLzBCsu;GZ212l&nF9{Qey){=N9d7H~MYKf3*(P2g@SNjR1cSb8*kN!(8^CNR0 zlaOdNb{dNxWM(!xOce`lEfqPwFN75W>qx2o0IrG`h5lrJZag9Z$?tP*qGd#}@id<* zxA#Zhxo@UCO3&j51;>IAz~>{||NPU~T+)Thb>g|?Az5@l>ibu!!+gA8cNaa5V2&H3 zI_@cqQ>kj0DkLA`5%l)0D^bbu)wQaHsk41FSmK@xZ@Uh5rnU<`>7|A9GOIb~J%NOU z6HMN9X%&wr($q2YKZ5s)!J?$f*yWeG51vke8s|^;V21#K+B$stO=p~PHQY-EKJAMNW&Mo z?~Ga>#QJEyB0G`5y7OhcN!BRC$v3Vh9hD=ls}Vr@mBP3dzp%@~iWk!orzpMvB4uQb zg$g%)G%7t7IGmo4PR_m@>C!l_rA+#@*Wgac-NhWu1}R> zMj3|;?HDMOzlonaruLA8R{7n1Mz7g@8Bm5ix3!8YoaT>?kD8=%5otV5L4#sGnJwTFb3sRAg(FRr%JmngCdWrbX`ah* zcb!^z(pIVOued{bw)TagHU;(yVqc7sP^7?WjR&?DSs_GxoscN|Xf{oRFJpKi{?jDxBsfX6 zEJoC=x8VtTom6XT>A#FC+}|LQY8bOQSmB}gGkyPeDaH4x)m?a$}Q;?E1 zv0E;8nJ+emCViZAgkk;ml`3#8Cf~?nGBLMcY$N2lzmob+vj5 zoZ9*jXY1v%Vz>)uP^nbTN(`($nxwG&Oy=ZG!ASe!%qWeI*rxq}`32S5P#CiYE|M}P zT|uit^-4+cAim_j*Qnnw*($Ud*^~w!>Q90R^9dlS8}=?$S~THKUNTm!{1$R(yZp#1 z5dbEZ%jw*oA}LjT%|Vg1vDQ9NpRKt$o$F@N2wz50aHo*}LycnI^ZeU_@`#ca6rU*X0ir<&UI_D}kK z#=2ix*~+%Go}HDpa`yr#qIlQ>J(DiO-F@;bWXx^%a*0rmbBkV|@}%ufLAnl_?M10Lt#T>EkhloZ-jF%5;yQ5sW9%-IGE%Bbg{&nJP|5r9Vdl|wt5mO<*l#&PieoNOW(#PgXfoU61N^*1 zC`ZlEj2om@)jDCAt!EjGYxNl!{~l6mkkceWu2!Bc1^5Z76V*T(cBoTmZhP_g;GOf~ zM-_}tbjEN#8KPkOi4h8z(e9LH>OI8xtVDN`>bH5-wDBC zeE9WelT*H~R99a9!|5o&L|LOXCa##M1Lm=<^3OGVZBZl}N*X-q2h}PdvqqllSquv! zA(tVIhP*9C2)QnO+7rgHR=1M(=LQ4D;@04)mD7E$0@+-LSLiEMe5rr?6shE))ANvL z+0>kgeI{oo?R)j3O&2D~P}$^9Gc-?N;YNpT@=Dgf6(jyFeG)cE)_0!DzFo>+InpKL z7mTBd*1#A<<(o92FVmOu9Rwga=rXSF}=Ju?{Y%9FI_%O4r22JWTc=iFsk@bjc7B zX*q$~mUZ5)|AL*mnAg3ry|Mi=Q0Jwo^M0fkzG@{_AecJ19s0sQ)+VQGl3eVCl4~n- zxyfZ?n|M~aw2)MOl#0)=K1Q#d#7L7p*(XYmoIA29xv{^sJ8}1y$DxJUiVsrHr(TeyC1B^S0R$f zT5OJK%?$;`|TvA-msKJK| z?Z4e^!6gm{N5P@EJcFjQNc@RV^&6iQ6e;nP8hpgVj@ohAlZqN+RS+VMZp(_j_$EBx zqu05uzK~Er7v6^QW7eSyS1RUR6jP*}Oz(EtsPt5w`Lx-pfNUaQ!#U$ zUkm{R&_?fN(_$!%raRmBTz6=iN)W=n)&&7Yza+*{Qe=7=NZ~vI z4DDB2nDa@-ley?}b6VQbp^n6cd75tTeMK{y-bt~N3;{_@$?nu^=mS!^O;U;Go&BXI zCHK@9gXXA_`wKpD_muMaox;}$5oXy|9C_B5h67ypUu-$>*J2}*ShomuVo7R>G!-a^ zEh-?QC{4dzj#DYSimeldjfgh8p9K|D3j!Lj9Xt4HpqqVRXb+E%^PE3XFRZ%ofr+M? z5A?fDiugFGt1+D|RygjYDTjPBl&Sp$m_Q~sP03?Zl0Yr#99U1D7dG?3`Jt-BhzZle zm<`^vwL8BByVza-W*f*A*k~MN(*3PVgf*(?>*iLaOu7ArGfb7pagoEIl@_h;bl26B zkEGjM@q7dyG7OCI_Q7y-o6+KAl<}qijaLg(g9bci=PNz-@yJ=GUoi?W_Buwf7$SX=e$HsbrlQ;zo`rfI_sS(%|y9i?#TtiWxW=h~Q zv@YV}sU*8-yK#@U-kqDIg^=`-KrXJ$9v|eXruj#Lnwr6l1pR!JJxd%wDUcymq|#0w z$<~R-9ELtoNk`UbcMg*29 zfe@z)T$O~L3LGU4$6465ckyyH#vQBItAu`3^%Zj~%E~9s6g4K*#qkgP0|Xv7k(l&L z2mzEv>SXfSUNRf$(e>_6soR9b8Q`5>M6S(;7?%1p&ni-n$dWfb%lQ+poQ3JJ!RhNh z@JS%XIf_VR$-p1U(awt`l_^R@?C|g%g;}23F{+HkDW~}6H2*F3$o7%OS3ThgJQx*n zZGsQro@|AcDxI-8&l=uUB^Y0m*njV?9ehuIsZbZ4LU>s5TMMH9u0;giAm5@LNMNI5tJt-5xH znT(17B5tC@$0li-MY_YW$2SPW&JP6RoW>BR0?R7mCd-t`nymOTJ7}QdC5ah@CeFbR znJ`Ip*PEeqxrtRfl#z4=$$@VX$V7a47V_z8ZVtWItgkv34o*&j!kf!gIKhyYAHR#^7{z~| z#%I+{e9byvn_Bnm+2*?o7BshIXgXD_LW39Orq6Zj@lom)Nb(F>wYx z#~B3^V7Vc%9N+|OB%F(aE;r%>IH|1x)YKDV$S6n9vJ09D`C@F7l_}-tUSVkYHG25u|yA+>0Cs@3BH0IxpLmAHQfC(v| z6{>9n=MED1KHPI7jmYn4CPVB@|32pQ46x{rGgx;OxwL^<7K;dRB$4s}eI_t(`F+ll zY++*aM0Df?E*jMuUnSV*dPA0wrm0>s!#}vqN(XTp9CGx_{{NAD%n13qyAJW5C1_~I ze7*7nzr`<>SL2(6Dw>hf7s{2=7o$x|TsYVBQ|fX@JE6C-=o-YS69;EZd-!6o^8#bH z33d3t_yM?IUQ(y@9YH)n(@prC{DSeO=rlWQBa38>5nmescVGAz&vXZQf~3iO_U(=! z8QP7oL^O=Wg{QhqY^HF#U87&HyoVCry?1vV{(|x5V0dAi`+Mx%)ZMLMu7V*@tc!^? z!K6lr<_h)%X*t$-0w+wAp7Z$ChMdt(r|WwQMZ0vJ#dFKlVFgMa)?3{&v1X00ORgPn z91dQR10uOF&wr=SzWB=!?AU%K8An6+48dd_YMyq1qCMt3AW@Eu6s2m*5s2G5rXY;5 zAuClSYa-d`YM?mA>txP6n)*ij@PORx?mmr2mR`}Z%6&vbDKn+TBXXg@nFAzLGB%Il zX=SS^tNhW}aBCb65H5^-_he<+BNM_#NRSLifMxc4++gsGbt`M%Fv8t@pJ-{1-9a?U zn;-dI358C|N`pTsV-TxviY0jzzHcf~cm!p3*UUk0audZZR^&E@;hvZ}^p>eu2Dq0L z@oP7-{mwHI?`k7>w7Czn{Y8dkKh}bM{JF_`hE}urD;usk`Iks+Fr(<1oz{}J5G16k z)u?2sFC@mYyeJ}Rp2quQYy4v<#y3JT=m$?xHWEl?Z zB~vla)nkEd59zC&NVb+G`?<&*^j^@L6!*0Lv9Fp}^Bxu(6=?wTGP5njIA%(jgp;hR zOFj`Z%}HL(EBY9-3`jgzTdE&G^9p=!+hGJp9k(>~2TV$}{>MI>B{o5|NwVe1n@X51(S+s>-8*y37K@1w zm@Gn$!ZLOpgd?E4pTD3XW2sI3u>YoK#-#jKR~yr<;Id*orZ>LFCyFqxOsqL%T(Vxw zlD!wn(>QEG95a^vlIa}#O_Vgh^tj2qnANlYAnBa?4M%E()q@Odx=Lw1v;o-un)%0T zGnZC}Dnj6L$y$y-x`2tm-IySf5*<59gQTDxzXN-?jfXk#D;G=^XUBNRK2Z;bUU1xQ zf@_!BqWbyXZQ=Q5qpi+HJmCi4X-x;2zT@YQz{zs6^5zgRRv z_y9nYK3^Y<>w7r1MK zBN4m);f4<9MTik(LTa(~AyfSfHp{M6@)#_7;AFj2Dz*4UHJ`uu+eo_m!)0rl^Z8^| zU1t?-+0*J}J4i40FZ76UA13~4vQvU{_C8ejgaKUSX*!M|kr@~!eb3<0!hCzaTVjaP zCN6`G%y?9an0N=axT(7HK;-E|4$_bpZ;4$?jD|18qw zl;6WMT)%p9QXOkF?53>;K&Ek56od(mcr^qt`KG;eU&62=oaFn1R0H2m2Htz_G*0eo z@Zp0&1yzKLH`}}7D;RKx#}S*}TIk{SQ!JaUN71mV*bS6blJVoaOz?xEqvi~8JwE)* z(Fpwtu@T7)jtV08RZNiC6eM#Lt+~<<4dvF08HGbNU2o)=Leg9G5LIwDiiNx$Ycww(K|?xU**a-v z(D`P3=6xfYpN#=PsCLyWjgCPCkld3N7f{*FNX2uQBbK+k8 zflNOzzuMN!kYZsL*40rmoTMKY1@Aqc0`pB_Bfr6Y{nBUI9(2jS=KX72|C_cN$>MUm zAZ6#_!og50LXBOFk|Ai&5q@C4`PjnoaUJUCiQ*M=2i|uGT5eZvVo_9UvxXDn`9`0c zYBOCfC%+4wvw6DAEibGvW};4pU)}A=B3CRa!r7=!n2eblio7*G$ij_AtK2|Z0o;A_ zIwj$I2RrMg+EqDr`<`z!mQwNb_bAr_{7FXPP0haFl33c$!Jb$xg<|z*3Dtpkt=5W# z3(4l8+jSVxwROEGOqJ_CXs+4%JIRr1I*hFjNmw51^)-I(KQlI#zQ(CA+T%K}Q(Kju zLzOnZ+n|q<-2x}5HxGW6upK#n<#(=Bg4|`iZrG=X$@6mIMBak$C|3!IKw%aR3_>7G zrB%9A8iGc$?mB>y63`!=&SW-SXV>lazMcHeX}#Pj&A9JI4#TSD*f|<5OYj`#&OA}0EPZ@c zZqYz_h=>|+5vKl?L zNWP?EuTM_Uf3TR^+ZeYsxu_16z46_3T`{3L*iY?U8zVkygX z8=NKxNKusE?z#qAMix!ALW*X8qLa*|sO-T}t{+I%dL9Nw#c>c^x2ZL7U$*>7llw+J0(-1Q2+W6-*X1tN^=ew+_qoW~ zNjH#Ah->8qTNO!nV7X|8d5Mb1_exsqA`OQcSWQ)v&t+u%#3K&U>0n6z%%$-Ald@-H ztI~aL5HbBJR8W`Y%g)f>1t+Q=W*z0;k9^eV^H9}ZyT0dGr4!FBGTU!d_}?$c|Hvk3 zw~uPK{R1wdnd%z&&0>ZfCUwASc0}Dpa*>*(k?62T>kewABkj<7AHOFtkhn0KSd-)T zw1Q8sch$!Ibgsf$w<5IYI>o>kOkwOg0!C5z`juK32X}ItU}?bE8V2b~y#!`Jouz4! z0xwBI^Vy7xS2~@4`mDm1dCO_^Fh|e%{Q?bN`qhGe4BuYe5*UvPmP}V7zUyp_aM8X= z>`WT2tZv`ojiH0&tz>{7=7=15RK1BA5lnPnq72MA9i;Z- z7XkPSBIL4UqXs5#`6S5qCy1k|-FwWc((z)Ez>rwqHcgOc4V32aH0{&2POD@P{4Rtw z^Pk{Qmk3O@kqonfY9|r>&g@!fRIYHE3-OGSaZE2=qm{xo%1-7<5)zi;3ao^*$0=BNF0W;fq&C< zDAVZ5{z~0DvFMDTrj|3DQ-bFf1PMHPoBj9CO*a&@GAejG%>f!<<`DL5tq3}gnLizdfq^4 zXbwkNo{@_c4x6D^D|PqHIYtP?p~;~*ggJh}A_NZtkESi5ywW4pAWBcnP3-hMHcQq- z*{{uBH!{;(aoF{FA_6rWvR^Koe*};CA=iR%4V|ESd;RLIFyo3-{*iX+o8ZG*0sVWJr*YBuHklHD)jfd!ocarop~5ezYYK%JG(5iFWYee4&4(p6{s) z5cY=*{5qK|T(n^l$CWuVlN$2g8swV_;P#rXz~)V^>bXw#HfQAfBO4p4$^J|HZkhlI zS1kB~^x6Ys^=W5&3?#DI_J02Y_sRR2pRjpUZTkmcm=QKx&8$vCF84gaC!!5Z4vf~k z@>TnsIChaWoy*Z(jA2XYLC{q)FV|>fnUz2%{J_#^@m^Q6l zjv1wu+IlV}8}T3%lF-D4q=hxXNKpxgazJHvlY*)`8%^{b0=}s7##khS3`!$J;tXEuBx0YEjO{pI)NwZxN%^XaTx8nBvfU zAlP)h>+*L@BupS%K4*_K5e|Yemtl*y2z4Ua9#8-Vr%W>iL)GSIq9%8g12Wcsp{v^}{ztt>&_Tq@Ijn=q%`wR_Y~@PYiZ)(%xyuUL%3`}k>uA^8vrLu{ zzkauiNF%YBnjv$R^Na}uR9L6<%9m$c*4K z!rA@V3RfqHAA+FemDQR@?dO3*Zc9oODUC9WZ9&8!FP#7NS@<8j1xNgGtV!O()g6Z$ z^Y=OSr|ggak~Tqs-QDGbtjT31UOtX*VSXPdO|t*@mi~p5q}9#c5xI5FaSiLgV5I2w zQWa$NKl-9`nj8}AmOsTPw-6+rLEPtmiA$j_J?DKrwFy2_&6#F~SniD(O}Y++Z}%-d z9+fitI1$EXBHNLS%zFA1ppV|G9`WwFm`={S2d8}e^ARUr)UG&>-3|JW<*iQl6>jxy ziKccb{2h1(ricE!`~b1n?KB3F|Iy;{<#0X516M8T2}W8_|D@me?733xo++SLH%d{V*FlX^zQsD{Go)SYpqw>9_AHp09%@D_GyXxKUW# z!xdu7$6Wq`;VGEx()R?umk=IVlKlk>llcn??&UNIuad0co9vJdp#RDqSRmBSH3q+dN5mq_pI!x?*|_3HSo@wY3wS|8-0w zz~u2=FCa0&{UofJp-p;AyEaRCv{*7_o`#I-~z5hwF))+~~8Y}tAb$#ZX+KdYuCW@?Y3tHSoyoy6ZBskTl5dH6| zI!z8=8y@m~I+MzODZBpz$9d#?V{7rGJN80HpC7>uzFZ36b9@gG#AcQN-39kf{hD$c z$H7^-!z-`%tHfh!)b_qteZbT>xT;=yWl#;-4<L zRGV(&vpWhqtFS|N`k5@4<8d?uQsY%oBr0=A9zF{b_N>xb{R8zHuMXL| z{7u}k<$zRx$UzN1D5l^}J6Pn{RUMo%-D?b)p_A-z=%8fHu|oJ)gceq7iFC;Ya}OMy znGIJx_?SLJ6C2*i+A`##5SWph+nzP3uM*!$pEhoUA!=lV`cmwpWa6^NG)nm1EGwvN0WHvBWYqI=5=KLPJZvPgD-}0 z@UQGqRZ7$YlmfV)s4&<=)Id*Jl-rR#|I2gW`{hGx znpG2AVpfAS{i3HC5`vXxh^BnV)e&b0XBR3 zjSke7Ea;2;)USjp*y;*lrii_vqn1PK>t4UGFx4}(ayk5B*?QI>LC(>u=aq)eB@QRY zB2NlkB(5jcVC07SXZ*>tPZLfWGEBZ=Ohf*$ZC-!Z#Xb<GnT?{^pJUR?TWQ+Hp zWMZ#9A-?y>o~bn1Sg^bVE>kqZf7IP?U(u(!pI^?Ng=&p3diXYI2Ff@{*$1({2a`EwV?F`MFX{o!j__ZJQAT2*aJwv{GGR}4Uc0dt zgWc3j53K%6L;(*Zn5i*FTy0x?5i-BOGP+Zh{;C9}I0e?BiI@PKM&S7ahhw$If>rYw zr(&l!UMm<`ydr0kzA54fZ~b0QI)3E@sG*gXI864C{EB?c zt=0wC)+~&SQ+)zKm9PTjRn>iIylZ4vCm7@TrphN$<_Du7;~G;pfnXJn4{|BR?@Sc7yY*L`PDM z9JKXR`&F6*7|G<&kX56Z#;SW_R@MLK{~dfZEwGJtxopz<&~4gC#7FN94MU(`=ejGw zDc`wGrK;HceUC5pX64HkhV;g`f<$dC060e(9EIQ8n5E?|f(7gQJUx6UD?A&>=qPo2 zGPs?{lCTqi*Q0lObsn%C-&flC50oIfYP^w1)5)IDza-z7v-NKZu{ zSF&%-8*?x`RswH2jUf%uUl2N(LnJDtnn&RB!#&PkwT|Rd3Xt~lAa7DCJNF1NrL}i& zg3f(PkvKNWTY}8uGPWXQjIj#nIk8SAm3eHfXfA@Y;CY`d0zhn!3^UOWzOJG6$RG%- zOuE{wZ^Na_SF@KMjgS{iFeVfH<&YOIan{F8<9dtBGlB76;&n8vH>O+uv*XH*gBc?( z80F=(J7|ta4S_~A)1qqu#fOcfdE_nI1b5BgO)&fQ0Y+oK8E@5=Sq5vJ@O4 zUsR)d9m6)cU(sgN&4$ke6 zhgDJV%w_wi9an|)8!QXg6`CnUsGXVH8T7>cu%Mu(if8*3Qqg%HD#5ZW?d1PJIlygh ze}sDe;COc!Cv)XxIl~(jl7v`z%;m6zCQjn_Fpu=T&m#sOk8Uus+K@vz*e~Z%!5E+= zaV@;c$o!=pb83 z%;8)4Vp46t8kTRLcf38*KYdkc2(9X4XJ?m;pToMZ(14efMlB(Wgy790K+)UP zyJzH<``7$~DvziCEvi9zGgsq4ui%2$4Y9brVX5qAw&8@YUk8_3hgf{Z2NNEIv#A6V zjQw)<%prmOokka4c(6ACBxjWT8~eq92~md z$i_zFj5GRbpSK|#3zuzsQ$otE(bR>2TPDL??eM2#)?luc=kABrKTy}+Kc3PctHICt zcj>RGZ$<`o&prFKdu(kG*@j8yX;oJuPmeNBo~h`#3x%GCzH6Y^BPWUrYO=4VHM$7= z&WarT&|tlr8-}Ng`!F{&ExzF#7Ic_g5Ejo17E2b}tZ>K`O@0*2_nvxYg_prSbU15= z&6()*(w?UbtET6`_%oh)wV#*oLI@85WV-K7hcSZe=RUtKdXO@1bPf3>(n?16WH3ju z-=QtPSbi5{OAT{fKy?%D<zJ$02y?MSC7Npso>DG3|<|T}OJ5%PVWjhF&(gQOHjQU34F{FI9 z7msSDw_fLdei9 z^2X{ySsIg+kk}H;_+fb&hgU;PvEV~3>nCnsI0Qu1(P9QcF7kW~s8|yE$Z|B)7o2N? z_!EeYUFY8I+?U5gO7m)OlbolJ7TFK|nyl|+QoGN*81oNeE`R>|-6ecq66H1ye1WkP znl#_rP@4mSxFs}hrF)sktLbCl~EaqjbEWoWB2R-VUx z?xy)lY!Y%iZq*Q16t&RCrnH~FG8sIz5TgtC27I@juAC$VkT@;Ge(!5Y`b0duHBleP z6e(4#DlY6kyI+r-hRgz5i+4W^3j1EuIUkirr19DhEs;85R*wdaVT3 zvHmYj66{c~|Im#IkzjZt(Lzc($!BxsxK8r~6T6!7*Wxc{8t2On9A0+}SqGf?Ix#g_ z*6Avi2BMpD*2vSIONxXsASn??ehMJ%E z3r%^er#7P*8TNInhzq?m^ee{ba~+%(j^H=elYf~;=U3+^)RccnQwSsB{D1c|A`nDfSa9PQCWOPQb_H#C30dFglNQ zOLpkKU}n3^2#^GHy=!eIe-HPry`|vYvEcjvbfM`CheP-Ke_x%0I0Bss;{fMbSUTCGJLWqLjS-R0*~Il?tuzrS`?8?QC*VFG_3F#fkR$C_YD_?_W{X)5%^ z|Dbm2SdSkM4syLX^A|$HGu3~A8;-$sN!9NtCCNrxxoxyw(TB42eM$gqsZz96>aqGW zcUoy687?kBrRY=uD;id^DXA`D>&qF(1O+L=WP8V6?qYz)2fNXeb>dVts-To6wbvG- z2M#n#YE~%ebAMtgavwPJ1Sp=a9$!0lMqrj$0#XLHn3;&SH>culq!dJoc-%KkH4C7xi~N?w?gBrjuA@ zuLL@N2x&d0H*HKA48Z3O)5;^^j>7drRw77<^vss0I0xx1z)@}c&OBNjQsrO0gPEzG z{0>V8)vmt_mL03AhTJYwJHE!*IjZq6tbbTx2K)W!ck%TRd(8A(w!BM zwl?iyj%>y6Bc*d{GhFmbOc3T`a%NO}qJ`cX7p?#Mig zz;QPD37P~a4wHHa+LnV_U~lq^$%q|y;MmZK0kD`&^T}w+q2J;gk4r~~ag2KEDAtHf zlm6s&3Tb~W1Fk<${9&gd84{B#I?z9_I7m-`J4lMN$wfCZj`bzCQXTFR+^%s={0?!V zZ0*59w)SBHMB;M7i-(8k5sYdg-ZN|NRq(pr_Qi~jA;#JsgkKPJcv({GbWKfVwVFgN zQh#aR-n!$$;8{Ii-L~-Cj3FLBK-o_aG(^jT5B;TmIBTA)Z|$~1f1jo^Nu%>5Lhz~g zYimY`jH6^caa=sIZBdg%k@r__-Ucu3O{H@ck;Ww}9E~$a6(O!+=I*zv?#AYI=H^vM z(e$jQn!%~D#KB-GrG&}gZrUI7cMQj>Y~>7(F8_vIn7y_?mD{Sc*WOfY-*JJHd$9&U zqzg|b5g6L;*-{VccfBZiuswY&);FJVTGsb!S@m*iUz;H>&GVBVpZP&2#Hkh}|bh zsbm7=3U=+3JmBq%5;o*xiJinn&|oY09(q`GM-Fvh%E)FrbtYcR@l@+i>ZPrQ>Vt4C5QZ=*zT@mbLwNr*KqZP;HWBQF? zCs}rxLr~+z@Tn+OEbNfE(2*O&hqYo`-TR)O)E`V@NEP^~Ri)mEhVZJOxolmJNrs1P zVzF1|KzXl5Pm2kY--@yriQmmqtCHlF5!?0W zKBf%Me3%%g_UkX#ddyA+@FtDOZ<3cBDpuF|oKl;tr?c8*&v2a~1EO99N=ZO;adq0S z+@))r#&rLp8>Fz~9)8qVFQx(w(C&0UiMbDurgZTkhb^}sA@kc>?>R7|iqq}>y zukzM&(|~ZCAE{dz^G!A-IWxK1GtFa*G~CW%g%3l{9%r(1bs-B~LkvH-ZgyN6a8rlM zIBwiWwNhaF8J)U};gW`8;u7R0nwqpR+eRBGjUXMKuF9ec9(_=%W>GnPuJfa}v0}hK zPz=GlD#BS~jX&)u-3g3>4y-@b_326UrZV1K|4PPA_8` zf&p=_Sx-=(04&N5m21^>r_VLtEeXWMClTK|a9vt@{hY9JN1{SxzQ>WoADz`LHW(_J zGE0<1+utE-*Om@Pl(|8eKDU@2%$^-+T6+A_rBw6F!5n|>!Z4c*4=2L0^i$e6x&Vm0 zzIE7g#3nxDn^$_dgM|5z{AMEg(HVC4EBP#Vd})HMJDIzZvy?}zF7|O)8YL8U?3}g#o)P$tb`o=%{ z=-f8~fII}cPu$`1s4_IO8k1_W2lcn;VcD>@mOzqtdgf%B&z@svdgHwGkbcF8G^ugNW zr(D*}SdpFDm4=JmM;EC+cfRXODH4P$sh0u97%uH=l@|>DZ03%mo>%*0H32dQ%0II# zS9Kin8ibd)ixHLAG`=)4gmIB|`%>4hT#wdCg|A?JBj-hx+CxAQaBW9KHXAimzkiZ( z*FoP8-q*Z{S?<(w(A_Qfn%3v_p$kN7j^t6_-|`9hKffDS>dD>OArPzE8b zrP3?aahH3c{a&l2X9Q(+(_xI0Rvp!+u2nFAnZ~k7PGC9z+MiNtNn+lwYSk;dT6&@6i-b0oM!J5$(gL4gK48)%T9Sr#TmbrW zEXW~2T#*y$z6zC}a>$tSaKJq3OQINYpCr0rgKs-(m76XT8jYp>>Kjpb{Uh&jX+9c`>>(;KH)I%nOO!DVs7VZnI zq=dEG6&=|>hl|9aN2GY$6iWxio0-3sMDnRl*n|chM<_Rk8gNX;gb7hN%A$lQg{hY) z4q59vf5Jq2s?Eygr#Poie)^_pX)sx$g= z0f|;}M|XCrikV-M6c1@kvPB)EN45<>{Z@#~9U~C$p)`FNT%Fgo+%!6uq6wDf;I{8D z7G7`6geE*du2SMLcUq7x`vEKron)zOcBK6-NRCGL3kk2nt(0L$;84X?iX_DR&2RS0 zmnJ#H^&j#EiJ0%Dk}(~D8M}3D8`4=ZX|l(& zC!Wf)w(mho`Hlc(!zUIJ$|8H6oOPz4gFtV)Wm?P&I&JE;mKaOC2ZFuFjMZoWP}Y40rHNZxO_y?0+qy^;{}c z(|8&xx_ri4C2_q~GBA#x9E@|pMm_7G%d%htp2YCzAvu|lR99YVWDcSfA}fnfQ+{Es zwuT5Y;z}=xpK+5m!ay@6>`%cBcE95{qdDQ7D3VA3ke zq>7XH);ZvS4Dr}WqKIbe*$h=5%Ka+jf2e*48~fR9EjFy4yL5!Kn9}YAUFM1h566n@ z#ShC;t00b;sG!!3Xs#Oep9vzrXK=7gX=eb~d>{PR6MP&Dz>`=iw28_Gw1?pRZ~!=p zLay`x3p+k1nccPx53fB?J`*9D@nnD58Asg)9Z%x{N8@$VYq9#9KsT4>Sd~~i3m@8- z)KRJ;ci4~s>>N~7?VU%?m8sS|Kfm9cxYcPY(EaD!`07doUG)IR?|qd(r=loBAL!G7 zIp24$za4M<1NOJgN2qf~4arnhLuwzklYO}elW{lC{F9;r^jJ)k(-!IQY`8z;Kj8?; z&(%m_nk{jkjP^xz&C^i=g-!Qj7k5#UKUqArw03LR-dYklo)HX1nbm_GLpI$nunq9Y z$k`Z2kvO?%RnJ1vyn8>V$8HJB==5*1IK`%f_AsvAZ|QChFAhiX@vf z0s2chhm35ye&VNMy;{pOwTw;|KWD-Y+g<8q=iM*N5OS;@65V)aX8=Fdm3BW?C6Z(> z%eBbBG=@$|7W_G$cX_Nz#;0s+?~xoDCF-kWAqIXEQsmFyH!70+buUCfy#^iWIWC8= zty0-`i2L6En!H@Q=#u5Fvuw(-3G9kA6pR}5uBo$BC^&B{;4$P?_gV|3WTSgL{Nl*4 z=oRMEXYyD}&ICN$E13cyp0A51)37RkT2%eYKx^1SzoDjP4mO;!WmFB$rK44)6G8=o zY80b*LzKrDS2z^)QSC2DdmUZ%MhAwUP>6YomV3TiAD3)t<67>;F2NvqmPeC}4Vp)C zCFa=2h)TXe3ya|>wDHN85%S+G86Y3dy6WiLf>n(Wu-dN-swzNwKjskix4G-4kqdvG7oayX@yQgVlB+r4v!m3R_zI#`aUr5`S-|}`6PdEqKxoe4Y^E=BTdQf({ z9!lLwxBxhV4nFBqHXp^2jht7`PARUutTf@r`)%-_Ry~KQ(H8i7$4EktA3zu0O)_MZ za|7!bZ+;QzP9B)tK{5IoD6UQB_C?c}e`*m@P|ECR22yyXzw<1n2P?)f?W&b02v@^q z9_H4Ke`_HbR%_y77_8W@YyDgNH;`tf_TZ!)KnT2DTFUy$h=}J&yBq_bjQTJK zx3W>>l0dH4FWOj3tx|;l$hhcTwV_~x^>vp2%|}hE2sNt)aJK}L(IPRE z0c~o_wyB($>Ez$eow8&dnnoBRaOxLDf4)y>8@i%$Oyh-t5oIT=2ZX5?eI9 zh3|xVEiq-$w*cHJ2#?q90{Qxm?O#S{5o4@o<)8#jo`p42Tc--=etu8~Z<|nwZV4uQBnD`E<#+IOJ zb(_2K`uEC9hf$@N3um-g%0B;OIe~ggF*(QC1#tLldcrK+xsTlbehnoSRecj&voUWI zs!>U0<($Z{g5(7y65-6O8Y(LbPnZ1qz`YvCmT z%bvs29UDeLFQ>DalNcI~F?qz+aWDvAuT9132pljen5G{UB1g41{-!D`+%^(+|KYq` zN0KvCT>{2&Yugk`Qk#^(W`0~LWa!l7FtWFMz+4duBzGVjS20@RXyKS1V*ftzs&*v1`9^AV^p|_QC*C zGiY;jurCg8(K!5%Wcgb(nWg4fjr_LpVDikNJ5o;c-;^neOFkOL>pn4e&?>GpD(0H- zckqP4ptv_`M7)F3y$pzVv>-N(&Wd3syL$Ctnds^$TcH4}4VF{t^@u&g)EgvY0#Bok zB^*x#``v1Kr7C)vogF`PHq2l%K;3(6Zj=SG%QtZ4zQ2+yL8Cd(h@hJGhpE44e4-cE zFNs<`k{kx%j@V!_rKc$k>~oUafPW48K-hDvxo^szt(<6Re!v$)D%6?!05uo?tyMpl1!aL+Me5=WSvW@TVPZJP;=?vMYLaU$bA_#4|5H{#97h=&-%(w)5Go ztk$&A*9qLSWTp5#oD`)v2V#_`=Kq`4d$8gl@0Ro=RJq5oKByE58v6wNkq3B&+E+TNhffFMGxm`>_eAE37{XI57TYCS}0F& zHa0+Bc3O+g6DI~9Q&t@+RbP=l^M_qox(YMueUZlEz)L$k44BC zpx;d{ox!Z12h?a+h7r>#CF%=Zf?UFwU!5OD%<&Hoz~ZMc1W3VFi7Or{pkIL@>JZzrqvVOg@3&t`SbG2DzRlK zMtAa2CCQ_-sqb>ZD`9H;#BX=GSD!&rrdqZGmi2%Uj&?^cd2Am z43cj?yKa|?5h^GgSlMZlDfHK#jLp_!Bzt8B*40S4cf9EZxsoA@_yRm3v-{)WT>&93@xhKPzaKf;Nx~mt84Zif?5fFX zn0E5wd&?_RmTN9v-%+f~Qx|9H**AHME?JqaYRrTC20c5ZJ> zz2rGu@x1<5ivjV|;h`|uM@H=3+r%Em?jR_|;lEADtLaCZ$==Z0O3ZkY=59+G-X^O|r4%+j*xer{XjzL%_Ht zaXQV<{eU0w=hWm5izme6b<7ZkbeUd#tr*!`cc$j_g<3IcV?^AA979Eps5QvwI$MpViD)f2Ba`#`S zM!KS}xXy8yp1&{tm`~@$*bkPNAyFeX9$GXW06IV3%7&Piu=lRI^MkNCvt>(@HL(@6 zz8AAhrpbOl*~_4k;F@C#d+}18C#2DTjp8O2w8q=;{QJ9y3<7Wb8%vy;|MqW{TWFf-U_swd2W#?fFme8@Jsv~GqqA~m==uWF)xx{o%B~k^;RV7MkrT5>1 zTSi5EUr?j;PU@yZah)X+`K&-@{Qhvhc z+a8`bMXvA#hDFu_c|c_X%dglTEbNtRPM{^P2}G;68Gs)rm6(eW6kN=d0gYvpuS;e> z2^NtaG5w$0u>22{OIH1d(f*Wxf5^tmvdLfP3*0SB`~yGjgd1UN(lB;q@Nqtu-6)t< zEp)FiVzR~Dj}56F*p{PDt3`=Evf;lG4cGdsl59hv#+>vew3Dr&Urp_A?Aeb$5J0Q) z;jFh=4k)M4(X(7*zha!T@yqo)`;jqFyALx*xww=>C@n|gy%`|^86Y=&vyrFy8HvQY z*tpYHyhN!p5uduP#pS2m@K3BGO|Lu+kD;IKvD@beyyv|2m&`lP_T3E8i__OtzitJ- z{G5bZ6E%b$8RB>=OyEDaXSd*e1xMUc#4q7&T!+L?@_C9+z@QS1)Qt4HCwyoalU<3K z`7?xgcc_?4av2x0yBhmv?=_aV{vrl*dFuG5a`KnISrQOxe}UAw2ukHVNXR+rWy|`K zuvq535#(|L_jr8b)bGY^I*gBh8>jUVg32`eo)y}1@UhKl+eBjQX*kjq(o)sH&HMzg z@}7@;G;wfoaB9SCKWceww9D)m)kD?i0GaH#C-dZCdzd9r^H9`I;`A43j>^J}sul z{OjEcAyRIWl!r!ym3D!z9jE@Ru-|x88+{?Un1ETI7B{)mb-d$Aq}g!n8^=>fQhB-e zqx^Cc4|Df{_JH5vg_5DwLVG#Gg-L{<0>bE=A~B}U8htR873>%=9anf^7Be3$P10Ucq=uvvV6;# z8ur&+3uNpjK^E8DnS6uPWt|D_HjtH(x=?KiYjtCH`P|y3l7pe0sS1Os94I)+#NtNp z(t-34X^@TNOo@0OE?%CLJ<6&z&!q3OndWcYrA1jX2g3YA`HfbXpJ-YtoD!4B;F}UG zzgf`U`Dc6M`?&kq{n#~J{Bs`@Zz5aQl_{?wgC}OERC&S#fU>684{mbYMZ&t!;Q@<{ zU{=3{OdeCefskV-d~(~PxaB4dj4mz9U%w|!G;${ZMuq&@;*Su~ZTRR+Ck5`{^|@t) znw(isP5Z)dkeRznY%x|cBaeNOj~KIFQFJ0TXe5${a25d1okW_@8T}?cNs@5Z-0isJ z=v0qiYj!wQuOO}wDasSk@w1c1=`v#qWK};%%KV_wzho)>h0tOTI+ES<1jF9?@_?)b z9egYMhpL{WZ~N)Zt?YSHr&o&e01Fpaf{1``#P>rf27O=t7kvOGk8&Y*BpSq1md8HR zHEciLxZSZ`COi|1cUTH3FKFc$SnG=bAeY3Pm)>JWCBSDnuaV31n$o+sDn$6j?LjV_ zVgu2yM#G~4y&$HQV3x>$F;gx3Cl57HuBTxCv!g%P^!Pz2#VHWtr)ME`h zdk{!4nU-WMQCt%xC$EP}+DaKZ4%p(E5oBq3VZdAw{e@Z!2q9`L28CfqgmXLEU?^bS zE>A>KxljT6x^CY{iFZvda zHD?1}Wnhj+PN|yRT+V9rpfIQJ1Ijv{_$`aNc8!;Q4vv1;?EvKhYypyPm6O-!qT=6u zS9#S_kCc%pKEWe`7oKD##ZmDX$7o8OES928HuS_VbH#kLh>SqoU$pO>>~>}+cCRx$ z736e8HCS-t<#7D6bjhvXSRJ&h6%V!A1WlC~OCDNQb*UY74~aJJ zF{q?yxk$=;*kq{6Mv@cv2?AOAgZGnzMw!$+sRk#c+=>HLS@Z8YPc`LZMy|)D7TQK_YXug)&Tzv+TOc5bA#uhS;?QcRBw>yOY5n+3LSDwIXqo zy>lL*{IX;a0%$WqPf-ftm3GP`k!KFX*DrqmCF_(GjX=pirc=JIHV^?PZQ}a zt%1l+CkGK>Slz9)#5_5GFZ*~Kn9gd@i0bv?g^xC}^C|B#mXrprg~C=d%G?F!N2x>j zl5d5T_plHh22c=P#Xa1`W8;Gxy;V5J zZ#yf)jzab+EC#DS@P>rc+X~>?r}!;zY81iG&f;#3tc@h4JGce* zJ$B$aTz-=>zTrijdJ+31Ltamh&XT^O&`pvzMqWdEJaeFjnlqn?GxXalM%Z?20IHv$ zs+WsasiJQ-WDBoMRIRH!X!V?WpXJ;t^NlEZlw)*Z0B5p(ni3eE9G;RWDSVld(95yl ztV*15hUEW?dZ!X?Qe;e+w(+OZ1Tv|IaBwwTI}(q zlI&nwxH&x)!9%Egx%kb9=S+=O%Jl{nh(MEm_+s1{$ibS{;RwmyBOONNm);Yj0>+^P zT;Onjjg@I=~eXkie1s5g+E8f4X(GvPCc?K}gmYjx(Fle(C z++`{As{&h1$z|y*>o@HAfMD*b@g#6_OHJ|}RZ zH!0ohN#4Ghf7;g|Sny}pPgLUj!W=gY2Omu-iF<^#<2yhui(FTLMWvsFr%{-jD~dBY z7RVg1(0>ctPlvH+ooT^DmViY-GNx`c8w4#^x|_}4-FM4jUixCd$BeU)?BZYoiib7( zo7FUY2&eTjMbDulh-m^#UUV{3e`|0t#tiM#J^(`d!V$xy9&SXsFKY450tu_YjyZY zHX7Loh9VNS;PA?;Ty(Ks2loD?^Z`ydDu_R_F@ZgqC%ic&NPy7u_Kfkql}l231K&=8 zo38)eu%jjUKfajlVcTTkBIKCFou_I!O}qE*#N5DpLeb=0tIy-N`bzas>x_kzXrRP> z>t*{@tqA16+XF*}S~X$}(&ZR~eFQU9f^d>dS?M#3-3=?C(I^qP)4`*LoCP|kzYi!etwNwk?-9)G#)$@0Rpt*t& zg-m3u9nGgauP$~C+$t01)wbU9QTG*?M$o0aS9%D1 z_&eR)z^W_Gd)>9ZH3#m?2>z)#R_tKpl-s3vGQfiXUnrr1nZ?MGGaV^D64;`69JZPF zee-Ua6`up2Wt_2#T=egEw_bNpx2Cb^#kQ6Dl-amJ5EQRc#tz{Defhii!Mi;PMMC)4 zjCsw6J=L6D{qwohKTw(JnD_IvIOP)4q#V?szM2g|bK91sz5U?$pZ{m`t@gLue&mHK zo^?OyRca;jI4)4(bV*w{Xr!5s4ag!|IrQh{osx)~CTaJ@(OHn+*7yq7FIv@tURqTd zx*>4VBS}>m1v8*v!7#P)iK?u*dLY}D&e&h3`FTxQK_W_MeQms-NSxEeAM(GCY>SPTq5)o5;+sJXsdB~BdZmfkEBsbj$qf6>F1|DC9W z8$c1NG+TMc4EnN24;> z9f@_c=v!W$>_fyfdEg-`7lPwqgiKOO+6H2d4K1LxD#5LFZ%7MEwwcqHWpgUoPdMqM zwG9`NLukCkA=1LNsp6Jq^Bq~p#7&210@JaaWeh8(Fv12c!7`2p+G=TJSk^61DP?pt zCy9}CUNb%U5ixiSva+ei+mNBy=Qcmxpj1oOkW?g)vA}c0U5Q*LJ~*+-}+V-H=)M{5tsq z{-NWZuNQ*r0Cm;;y_N>m>^QBrp;2OK{SlM{a6UyElEo0&%=nlJlB@)ExY_+`nBI~%j*K!{o!b7XBLW6KNW<+r1H!#jU<60=)exGrYa?l# z4-o^s)`n3TFs@eY#D4r2d$27&qLHyJdq2i%Jgu(*b6jF`84%AU+h^rn&8|LTOMVFO z;@mw?IziRu94szV42F6USlDd3`G$xe^i|!VDwaROsGtExmfig@QAj`0u_Jd}(@B3;;9lZXWMiF7D%tsa1Yxx1h#Ql^L;sx1Bf@&~rqZLy*wNjnw z^=)Ph>d=S0`|-n~@0<4h*JP>BT?`wJrgww0PNE(zy@@z*6Gy?kborg>!jV0Wb+aJ1kUnAW^jS?&>b>eYt^#U_<5QyD7 zw-h&191--}v57~srVGs447sdbSEQxi8*r`hhO3eK%dl;3cf^swfv5{1!`GEzDB^dz zlms=Jn{qLVdLtVn=H__3U4k_5;!k*`9sIO~-4*{&fU@F6@DH^{+3E|JPA)OZR{yFZtqkDeh_gIY%s*IP&0a zJZ{(Z#~4T5h1J$?X7y=;`WT)nnDyXtbo{NeoDQATCLxj73R7z$L#iYgYn7;(Z+`rf z8GjzNrv#L!Kao1o{>ro<^bioD!<}qrCP}bJHj+>y6Rd#;3H5VMrot{3 z&BZ7FY`3H^S>K*D-*vMQZlA_Xv#0Ak*r)$X+1}gA_8;Sz@x(`}!-nd1fo1|O?OnlH zv6ZrC+oazc@j`Z@Hb5BkaY`~VE4FYLOg}U#TCW%t0AcJm#_ru1MUut=vzSq6Gfu@H zxz{dPN7Xa#8_0N>UPRPpomOKSLaXDko?`v*4ut9$p^g|oIS4Y?*^^kR$NbJ*#u^M{pz zs@h?WJ{204)^gKqEHx~Y>~w}RE70f?6d~Z_?hSHkF)cw5BY+>lzr#mzg(2+S&A7(y zXTtZ!MZ-hD!urz?F-5?t-uoL>uE8k!e5B5{VNtzpJT{YC^1)dF%&Y%3w(X8;$=GKl z1{CHNR}N5qAs)uEz--Z%+(;V0=MC4 z7ggwN;4vOq=Q*_AdM?*(I*(sca>BY5UT{Y2|Gw)O>L}IG(+bK zd#`&vLE;cL1D-aO9KmZkXAschm8E&zPH_psw%aLcq*OZ?yfUb%@p}EC{KPj*BK#+X ziE;=-(rTYWK(5$Q!#plD+iXcHq10U*C`|akZOq%4fwhmqT{NceUX{;llJdy7Zm@`k z!TizXWK2|ZqDk7)yr@|{QTzkRq=I+u%|TlOoB_wUQRMkiGhtid)qZBD=GZWQUhb5$ zyHQ%WThkEYooKll)&Ms{C7HK3dS|;zfvu3UC%ID)v~qGM=VZhU#FCZA8MQ+Ve!T=q zX`;#}xMRtl!YZ#Q(TOH0hfG^0!B5S1*`Pyzk$}}~ZlSLjuQTROR$cqfXHxQH=+r#F zTfGPw2$xgLo^JHYE_?uN!>oL-RELp*u9qO#r!R4wfKU<%EB>E!yP*{NfO9pvfIF2- zb_ig~8axi8iXV=Q^E}3>UJm1>Z!r0-AYAgZt5Y-_$$kzOt48^kn4;{8j+3UKRB3Totl_8&C+t~tQj`o>qBJVNlP0am_EeV~B5u`r`z%9UUCpdI zVfXaah%DUA4KMsAk9=t5ydGYg@Vyw1Y;cd-6usg^uo59&9EtRBg0amxG`dOO@b}}r zhVI(mdLhSI@XeovaJ?HIjiy3D zIMKeHB=06|rBxBh1Ks9{nub*!g;0bcGA?|G-o&Ug>+))>Butzg0K4|MIapfGakDspBv<9rFvE@w}}v;88G|3QMd~CP$SF zTaDC6EG$BuA;&hvAWREYsRyOW5pv_QgnT}_1SYeBZruKag#BObomEg=;g*G)K(OHM z?(Q@i8gJYsIE~Y|2TNnYt#JZ`03oI3JnS0--=3$=B+j%&3 zYVUvTz5Z_nLH*VT90^07%AL9x;haDW4>m8oK!bOMOiK+$2|)NLdU*`98i6zlH$N9+ zy0-v%2g6(k85OB_M#vir;u=+hHfq1g_a-Nyw7dibeZED8#rh_>2QDd61_UI@ zlFku#>zzdX8UleL38FPEMzhpB@mcdv+*tOX*8ubz*1BUv0W*U%8hZcCRUfq^?y{#*>n3lGiQG6M(J#1UlvYFaT z(Y}3@0t>E{AfWXO09E~tJ&w`dNYi8idZPC5Ilv7$S1-am z&?>O(N@100Zzgi}Dr)(zIMW&$`MF2AZ^HTp%ZNZ%zp*F01L4HOYr}idH@=oUuRyd= z&;TQDMq)_{i_@rTCN;!y7RJmdo%)u$Qbt&b$HR4h;3j0$Y@C{I#)fv{A`Tp-2BM-) zWw$=id3&JLX+2{~>VqDiq%4Or83k8--?^O}GP}HPS!v+8WLMjf{q?Bk;&leHu*%N) z8=dQhM>t9%GD{og%NuZGSd*@OgIOiw&_&tbjgs?Wv(ofXc=D(USI8wyHJRhQ@f?4_ zJ%cGbFG*!OOWPGh@8nSAmOE{mdGu{_ZpK^a;GH}0@k*FMHLvVe3O z@Q_&m<6UuWzCWrz5q$=fmS)SbIll$n9xa{BHO4ZVWhty*JpCfyFIzs5h@{50I29%Oy6Oif5Rds&95-XY=~p4=!Lk(AW;S$u1fP{ z%YA+Kd5wWE$ng;Vr%u{wCOyom5Ln~mt8h$%hc?^uOnEFNuUNtPdSCx^Lk5&br|+#i zacyPiwAF&!%^YPFPh6|Qx5X|VtG%_z$+m{;uiV2fOaeAp7Ha}#3^a5CA4Zf&OuJR5 zv}ek#;GCUE3}G-C6hpuvo|jU{DPA+L70f~a7J)ZTV5}lRdN^vGF{AKm*L4r;{%bTf z5FT5Amzd;h(u`QM2KB7yJ)B;_?13_7sDHX;2evk)fXEkH&F+S(Wqu+b8JP-dT>VH|q-O;Hs`! zX18C__mXHu>kr}?PUlI9{w^k$$~KNtQIHSG3HdYt$>->pe!Hr1^2Pfg{A!(!=64S$5YW4yb5$pqAvd zc8iiKT`&7uYMw^y`5zfQlaVjA#eRiK)J`6c=#{F0{KY%bRIPi($r1v5E!W^7QGq9O z2~+EtrflwCX4i?|-;%ZrpJg`$^L&13|4Mh|=r^86n)qlcP7W>bP9;*|8h~cSq)Xrv zIA&_%8`laR^xxB`2!h`k$$~K4)08$p=?;J;JPYE@Y$wbD;D4HVHJ`|Dx34&p!r{`` zCV4AehmO%20hCs8-p@Z1wq);%69j+Qcji){t?N;~z7{40y>9CeL)E8{a+7S?pFZ}O zT0c(z5v;UWe47s1ELk*j%IV z<5PC1$V9j$nio*=*$^7sgB&=yUmuIsR93iUZF-a|UW6%=rhE%r6@}!S5EH%`tquH4 z4U;di`A}R5T8d-azg3E`>rb)-HlXoXOzYh3S1WNF^qjEU^ZpJ%O1auzFIlCZkBiBM zk=EW6-j4OAS?Uz>0j%+`w6YX5(2kOE$cUKkM(L2H;?#}=>UVo$cM2&{j{f(z@8Aq` z_NGm~WE}f?T_aw57eR|d)){=vjG2NXon8q-G{47Cw6WRoBU(p-TG>u47^*}vKds(? zX*z^x*NjOTN{nMxO?C{#R&U0$U1ORN;5IKZU>xWY~NKDRg4A zVSMzEd))jbr8N8CJFyPENuAFK7f!?KhOnam)Q_+Cm#y6g_Mu7k8^ndS-xO@B{h)^P zXid-*4rOg(l%yGfG;D7+4vseh-T}gT*9Vj6w6tjck!zDUz?2a?SR%iO&J}EQAvMq= z>{=ox~R7h>xaB-OD^> z(6-#rI}gjh0B0$hKS!R~H=+95GZ;`4bVzJYiP0{|Zd zJ`JGF46y`gBxXmeKA(th$Jr#1Tg^e2T|k@VW+0|aR`tkVe#Xm}kdSwjtowE_P8Cfs zQ~mit`$m0+gY@~6FAlv8d97s1yyBJI)5c48;>|CAPPn40yB8qmTa7uE#>on^4h3_k z;%1Tw39aql#_8;w>s3cBT#;*i7{_2@p7aBw_g{d2kt#mSltaLuV162x|3;2^bJ)Do zmK7il?27+znti$3puK)EW|e5}@k6Drdxa*cu&PsHAhgcqlsA14R4Q;g4Bkd;6eXA5 zaeS+l8rxOI5c?wzeg>t$%EB`siNfj^A{bxYT%A8p51Rsg+>5MFcSlW@(8n8=L`&A2fVG zYZpUHeKw|4OaYIpjTB6OmVe53cVn;(%w5w(A?#nk3|y?Yg4e*r-+wiH%>dSyu3 zZ?(Hi#Hq3qAsr2P&ZX{rylW-y!$ZPs;yK{07OG*Sz*VF%$AXt~A@%K>x!Mh*_0^<} zJUSo+>h)nE@#;z7Y1o|)uXwDg;Rr+XA8>NQER^J;L78~m(y71Ess={jummW z2F|km)G~8PMR?N*Z=mMw_>KENHSp_FYHCE6HRl+Q^Rs9~p9dzF#WyDr6B7Ju-)3xS?fb(uDzZx{|!gAiu;7_M|1D#XEXKvFzrwMpF*+s;#n<{zFaz zjss2FEnZ9ESoSsd0MEVjmY*>lT4PKZSSuva#>+L`u5B4AS4q0dkOhdDj{aP*60~q! zt4SXY#Ai9_QS3vx9(o6*5Yy0SuAe|1o+?c-i;GR>r+5wkSkAr0lX7D9J~IyX%(P%Q z9=kQwU5M%T>X86gOa-C29%?ssnZ~mK8*Pe&3)NHf8LN^ZG3T(v*j>Lk)e`xGkLAx1 zj>BEyOxYv2k*FltLyTn!blU&0>ZGNYRH|=L-H%mY=0pi})6p z=;E+v*34X9Iv4QSJSvtVBA(q<4^6%)4b>CeX;0E!7EL*41@Uff%a9oZ{`84wyRLSI zGdX_CeTL&1E?EKqJjLe6EVkBY5I0>2Th7XW@4IsX!*PXISy{b--Ha`Y&JD z*Nabg(Q(&08>msI)!rL)7V$S5Bwqrn`Y^CW%^Mrya)b@f zCj_WeZwQ(@>yiFkL+b99USpn&4DD**pJ^3J)g>x=S@m9h)rzT)WBSPW)=VqF%mo%(K5$6>kwvc1 z{Q1OrU3Hh1uC{F1(kp=8`yoWSX9SBELSvcE8f&L|3Z18axTk-R$aairC*jU@?bUjd zqbgT?>&oi3{o(_V0`ErcB{oi|MB?$q`ZS9!CqT7OVMb@^TO(7%W+Ne=yg1Hlk?ZgG zwL}d>u~-Z{WZGHI-W3e{>bgBAR>x($`9Ht3h@V-lg!g?E{iAWSju3R3;vhKc5(8qh zQWB)c#wCv^;y=TD#Z_BXrc1-1k8Hv$RhcWEAKdcG>`$vJSM(}5LznQo|CKA^*m6L) zBnAqFBaLE_-`qz?T5&44eDR04qm~zLh3`QpRYGWASlBV2g$UQ}3OtmrmJ}GIcR{ z%~?KhGlw2m*e`FsG7z9a93{0`%D?ru-SYhWl?{{>Y%D`ia6Q%{*vo;Bm^5t+SJ@1zS`M(t39z(-#Tg;-ZlCvUO#AJ9!X8P`M3mQffy zSel3rO)nTyZ8;MTu&M%EeUA2oQRR}x-w3iB*{>v4vANNuaY+y~p)_(NAkqfcw3tt@mqJky8-;?E(L`ATm z(Q&DARQqkpj6jSc2kl;B^eH*tc-)Txo&0R>L=sWaI%Iw-xy3dGS;DUdKlt?u)_&5j zr7)2*tZS6vqIX)0*9&<8yD7_f`DZk!kpA2?>2017G_}GBxYk${onggyY4rdiPlfAMOX3^w=}iM24*VGhgp^hZY;3zSd-lo-Ag!PFDDh zRr!Rz0=;Id;WXo~Yj~&DUpiV`^N3K0)xL;rh?n1Ku;-`I zrUBx+Mni5)G+?)7^o?VHNhu$OMuS%$Eek}hK3qT9xg{4m=?MJ!S6u2|($hp?b+6;} zU%)TU)4ChOy*j}-2kr?0Lnd+zKL8$Pzl(;=@# zP+NHFsAhr&evKyxebW(59AkfV&mJ+PwpU-nu|!oqAA{O;0N#;Hl6| z^s_jLScpZ@K9%D6dLk*$~NZ^bdw-RU+? zaLalqLUqBcdd>vzN;;zaN}_WDvSu z^Zlst>A)>RhFMJSP;v9QOqWTR^fc1^vt-8kSanc`T(BFT|5kgjO1&HfuVQS!G2S~M zyyyHaTN9Qd5_GCCs$b_`(YN8QS~0Y--so1ABaEbR2A z!xbEICXcH5SS*<)y}ErX_>>YY3FtV{f_6L*B5B)ph{fdU9pcX-yaK<+vM~(QYL;FvJg`vt5$&yPKwiMmS0n8_DQ}8G4R7%n}&S0O7KdSea_3Zu=?gf7vC~ol(C!)Pd8sw zI!tQH=MfuG{-Edt#gPtb?036xR}pq-m@^UhbXGqMlvbRMu82G8$he=zDe^(qNf6_o ziKH3-+~^DNKEKXesI9x=X#$)!Ypi`Y}Xl=!C~i#JlAkKFk4`g-kBy({dU(7CANKHYOtA3i0g?( zumFLRBhvp62pU&G85ijs0om;;T5Te?c{V!%Oe zj{d!nhIDb-*x&*Hyk3bkWE_>rkd*hU=^yD8nf?2*2DzS(cDVj6MIC)zw#q*-)mbDK z7b;EVc3rhnG;yYo=jGNLw_=>{Q`Cq)NGjg=Agcsdt6<6+xFSK2j5KuSnti=A|4|$j z<`dP(Nyut9o1s&@j7V<9{;Xs!>%f?PvANOvbJ2r%rp}M}CE|B1k%xU#6r7S&ONB7z z@OXqP9TAg0BL1pmrUrWthY|uc0(2?fttHj)dG;@(~!Eb|es#$7`&%KMqCm>*{$3$vI z6+2J=!wFypx`UcXaZba2d<`)WE2GsDX{_4%4`Fcn7tkVugxs@LUNIdy`~|EVyJ&CI ziQm4w`8f3#nT=Nzd3Sp2lsVlmq-~mK4FBOj(~+=aZF8#um8h0yM@z2U$v@(XH1PZ( zGQrIh?j>}zC+Jupk!j>);e(C`E=cF)ekE4{HzKTS`&QOhceK zq|^Be$Tz=DTGT4M6-{;!kg+8c6oN2QD7Gq0JX*J7i5ccGN!HdRzY%a3C5?u8P{e^`y3$n_eDf5S(NVO|QC(mJj6|b!NIY!)Jb5& Date: Fri, 23 Jan 2026 12:49:34 +0800 Subject: [PATCH 06/83] [Test] Add precision test cases for Qwen3-Omni-30B-A3B-Instruct in CI (#828) Signed-off-by: wangyu31577 Signed-off-by: Hongsheng Liu Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: wangyu31577 Co-authored-by: Hongsheng Liu --- tests/conftest.py | 35 +- tests/e2e/offline_inference/conftest.py | 2 + tests/e2e/offline_inference/test_cache_dit.py | 75 ++-- tests/e2e/offline_inference/test_t2i_model.py | 62 ++-- tests/e2e/offline_inference/test_teacache.py | 75 ++-- tests/e2e/online_serving/test_qwen3_omni.py | 320 ++++++++---------- .../test_qwen3_omni_expansion.py | 63 ---- tests/e2e/stage_configs/qwen3_omni_ci.yaml | 4 +- 8 files changed, 266 insertions(+), 370 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a8edf2d0e..09696658d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,6 +52,10 @@ def default_vllm_config(): @pytest.fixture(autouse=True) def clean_gpu_memory_between_tests(): + clean_gpu_memory() + + +def clean_gpu_memory(): if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1": yield return @@ -528,7 +532,6 @@ def convert_audio_to_text(audio_data): temperature=0.0, word_timestamps=True, condition_on_previous_text=False, - initial_prompt="Please transcribe with proper word spacing.", )["text"] if text: return text @@ -709,31 +712,5 @@ class OmniServer: def __exit__(self, exc_type, exc_val, exc_tb): if self.proc: - try: - parent = psutil.Process(self.proc.pid) - children = parent.children(recursive=True) - for child in children: - try: - child.terminate() - except psutil.NoSuchProcess: - pass - - gone, still_alive = psutil.wait_procs(children, timeout=10) - - for child in still_alive: - try: - child.kill() - except psutil.NoSuchProcess: - pass - - try: - parent.terminate() - parent.wait(timeout=10) - except (psutil.NoSuchProcess, psutil.TimeoutExpired): - try: - parent.kill() - except psutil.NoSuchProcess: - pass - - except psutil.NoSuchProcess: - pass + self._kill_process_tree(self.proc.pid) + clean_gpu_memory() diff --git a/tests/e2e/offline_inference/conftest.py b/tests/e2e/offline_inference/conftest.py index 8bae08760..276fd8844 100644 --- a/tests/e2e/offline_inference/conftest.py +++ b/tests/e2e/offline_inference/conftest.py @@ -10,6 +10,7 @@ import pytest from vllm.distributed.parallel_state import cleanup_dist_env_and_memory from vllm.sampling_params import SamplingParams +from tests.conftest import clean_gpu_memory from vllm_omni.entrypoints.omni import Omni from vllm_omni.outputs import OmniRequestOutput @@ -335,6 +336,7 @@ class OmniRunner: self.close() del self.omni cleanup_dist_env_and_memory() + clean_gpu_memory() def close(self): """Close and cleanup the Omni instance.""" diff --git a/tests/e2e/offline_inference/test_cache_dit.py b/tests/e2e/offline_inference/test_cache_dit.py index 546ff7436..9eaee00c3 100644 --- a/tests/e2e/offline_inference/test_cache_dit.py +++ b/tests/e2e/offline_inference/test_cache_dit.py @@ -40,42 +40,49 @@ def test_cache_dit(model_name: str): "residual_diff_threshold": 0.24, "max_continuous_cached_steps": 3, } + m = None + try: + m = Omni( + model=model_name, + cache_backend="cache_dit", + cache_config=cache_config, + ) - m = Omni( - model=model_name, - cache_backend="cache_dit", - cache_config=cache_config, - ) + # Use minimal settings for fast testing + height = 256 + width = 256 + num_inference_steps = 4 # Minimal steps for fast test - # Use minimal settings for fast testing - height = 256 - width = 256 - num_inference_steps = 4 # Minimal steps for fast test + outputs = m.generate( + "a photo of a cat sitting on a laptop keyboard", + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, # Single output for speed + ) + # Extract images from request_output[0]['images'] + first_output = outputs[0] + assert first_output.final_output_type == "image" + if not hasattr(first_output, "request_output") or not first_output.request_output: + raise ValueError("No request_output found in OmniRequestOutput") - outputs = m.generate( - "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=num_inference_steps, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=1, # Single output for speed - ) - # Extract images from request_output[0]['images'] - first_output = outputs[0] - assert first_output.final_output_type == "image" - if not hasattr(first_output, "request_output") or not first_output.request_output: - raise ValueError("No request_output found in OmniRequestOutput") + req_out = first_output.request_output[0] + if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): + raise ValueError("Invalid request_output structure or missing 'images' key") - req_out = first_output.request_output[0] - if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): - raise ValueError("Invalid request_output structure or missing 'images' key") + images = req_out.images - images = req_out.images - - # Verify generation succeeded - assert images is not None - assert len(images) == 1 - # Check image size - assert images[0].width == width - assert images[0].height == height + # Verify generation succeeded + assert images is not None + assert len(images) == 1 + # Check image size + assert images[0].width == width + assert images[0].height == height + except Exception as e: + print(f"Test failed with error: {e}") + raise + finally: + if m is not None and hasattr(m, "close"): + m.close() diff --git a/tests/e2e/offline_inference/test_t2i_model.py b/tests/e2e/offline_inference/test_t2i_model.py index f491fa72e..09fadc93a 100644 --- a/tests/e2e/offline_inference/test_t2i_model.py +++ b/tests/e2e/offline_inference/test_t2i_model.py @@ -34,33 +34,41 @@ elif is_rocm(): @pytest.mark.parametrize("model_name", models) def test_diffusion_model(model_name: str): - m = Omni(model=model_name) - # high resolution may cause OOM on L4 - height = 256 - width = 256 - outputs = m.generate( - "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=2, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=2, - ) - # Extract images from request_output[0]['images'] - first_output = outputs[0] - assert first_output.final_output_type == "image" - if not hasattr(first_output, "request_output") or not first_output.request_output: - raise ValueError("No request_output found in OmniRequestOutput") + m = None + try: + m = Omni(model=model_name) + # high resolution may cause OOM on L4 + height = 256 + width = 256 + outputs = m.generate( + "a photo of a cat sitting on a laptop keyboard", + height=height, + width=width, + num_inference_steps=2, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=2, + ) + # Extract images from request_output[0]['images'] + first_output = outputs[0] + assert first_output.final_output_type == "image" + if not hasattr(first_output, "request_output") or not first_output.request_output: + raise ValueError("No request_output found in OmniRequestOutput") - req_out = first_output.request_output[0] - if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): - raise ValueError("Invalid request_output structure or missing 'images' key") + req_out = first_output.request_output[0] + if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): + raise ValueError("Invalid request_output structure or missing 'images' key") - images = req_out.images + images = req_out.images - assert len(images) == 2 - # check image size - assert images[0].width == width - assert images[0].height == height - images[0].save("image_output.png") + assert len(images) == 2 + # check image size + assert images[0].width == width + assert images[0].height == height + images[0].save("image_output.png") + except Exception as e: + print(f"Test failed with error: {e}") + raise + finally: + if m is not None and hasattr(m, "close"): + m.close() diff --git a/tests/e2e/offline_inference/test_teacache.py b/tests/e2e/offline_inference/test_teacache.py index dc8004f2f..7ce915b96 100644 --- a/tests/e2e/offline_inference/test_teacache.py +++ b/tests/e2e/offline_inference/test_teacache.py @@ -36,42 +36,49 @@ def test_teacache(model_name: str): cache_config = { "rel_l1_thresh": 0.2, # Default threshold } + m = None + try: + m = Omni( + model=model_name, + cache_backend="tea_cache", + cache_config=cache_config, + ) - m = Omni( - model=model_name, - cache_backend="tea_cache", - cache_config=cache_config, - ) + # Use minimal settings for fast testing + height = 256 + width = 256 + num_inference_steps = 4 # Minimal steps for fast test - # Use minimal settings for fast testing - height = 256 - width = 256 - num_inference_steps = 4 # Minimal steps for fast test + outputs = m.generate( + "a photo of a cat sitting on a laptop keyboard", + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, # Single output for speed + ) + # Extract images from request_output[0]['images'] + first_output = outputs[0] + assert first_output.final_output_type == "image" + if not hasattr(first_output, "request_output") or not first_output.request_output: + raise ValueError("No request_output found in OmniRequestOutput") - outputs = m.generate( - "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=num_inference_steps, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=1, # Single output for speed - ) - # Extract images from request_output[0]['images'] - first_output = outputs[0] - assert first_output.final_output_type == "image" - if not hasattr(first_output, "request_output") or not first_output.request_output: - raise ValueError("No request_output found in OmniRequestOutput") + req_out = first_output.request_output[0] + if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): + raise ValueError("Invalid request_output structure or missing 'images' key") - req_out = first_output.request_output[0] - if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): - raise ValueError("Invalid request_output structure or missing 'images' key") + images = req_out.images - images = req_out.images - - # Verify generation succeeded - assert images is not None - assert len(images) == 1 - # Check image size - assert images[0].width == width - assert images[0].height == height + # Verify generation succeeded + assert images is not None + assert len(images) == 1 + # Check image size + assert images[0].width == width + assert images[0].height == height + except Exception as e: + print(f"Test failed with error: {e}") + raise + finally: + if m is not None and hasattr(m, "close"): + m.close() diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index 33a05c1ce..c458a1686 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -8,21 +8,22 @@ import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" -import base64 import concurrent.futures -import ctypes -import signal -import socket -import subprocess -import sys import time from pathlib import Path import openai import pytest -from vllm.assets.video import VideoAsset -from vllm.utils.network_utils import get_open_port +from tests.conftest import ( + OmniServer, + convert_audio_to_text, + cosine_similarity_text, + dummy_messages_from_mix_data, + generate_synthetic_audio, + generate_synthetic_image, + generate_synthetic_video, +) from vllm_omni.utils import is_rocm models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] @@ -32,114 +33,27 @@ if is_rocm(): # ROCm stage config optimized for MI325 GPU stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")] else: - stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen3_omni_ci.yaml")] + stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")] # Create parameter combinations for model and stage config test_params = [(model, stage_config) for model in models for stage_config in stage_configs] -class OmniServer: - """Omniserver for vLLM-Omni tests.""" - - def __init__( - self, - model: str, - serve_args: list[str], - *, - env_dict: dict[str, str] | None = None, - ) -> None: - self.model = model - self.serve_args = serve_args - self.env_dict = env_dict - self.proc: subprocess.Popen | None = None - self.host = "127.0.0.1" - self.port = get_open_port() - - def _start_server(self) -> None: - """Start the vLLM-Omni server subprocess.""" - env = os.environ.copy() - env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - if self.env_dict is not None: - env.update(self.env_dict) - - cmd = [ - sys.executable, - "-m", - "vllm_omni.entrypoints.cli.main", - "serve", - self.model, - "--omni", - "--host", - self.host, - "--port", - str(self.port), - ] + self.serve_args - - # Helper to ensure child process dies when parent dies - libc = ctypes.CDLL("libc.so.6") - - def preexec_fn(): - # Ensure the child process receives SIGTERM when the parent (this test runner) dies. - # This prevents orphaned processes if the test is killed unexpectedly. - PR_SET_PDEATHSIG = 1 - libc.prctl(PR_SET_PDEATHSIG, signal.SIGTERM) - - print(f"Launching OmniServer with: {' '.join(cmd)}") - self.proc = subprocess.Popen( - cmd, - env=env, - cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), # Set working directory to vllm-omni root - start_new_session=True, - preexec_fn=preexec_fn, - ) - - # Wait for server to be ready - max_wait = 600 # 10 minutes - start_time = time.time() - while time.time() - start_time < max_wait: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(1) - result = sock.connect_ex((self.host, self.port)) - if result == 0: - print(f"Server ready on {self.host}:{self.port}") - return - except Exception: - pass - time.sleep(2) - - raise RuntimeError(f"Server failed to start within {max_wait} seconds") - - def __enter__(self): - self._start_server() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.proc: - try: - os.killpg(self.proc.pid, signal.SIGTERM) - except ProcessLookupError: - pass - - try: - self.proc.wait(timeout=30) - except subprocess.TimeoutExpired: - try: - os.killpg(self.proc.pid, signal.SIGKILL) - except ProcessLookupError: - pass - self.proc.wait() - - -@pytest.fixture +@pytest.fixture(scope="module") def omni_server(request): """Start vLLM-Omni server as a subprocess with actual model weights. - Uses module scope so the server starts only once for all tests. + Uses session scope so the server starts only once for the entire test session. Multi-stage initialization can take 10-20+ minutes. """ model, stage_config_path = request.param - with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server: + + print(f"Starting OmniServer with model: {model}") + print("This may take 10-20+ minutes for initialization...") + + with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server: + print("OmniServer started successfully") yield server + print("OmniServer stopped") @pytest.fixture @@ -151,15 +65,6 @@ def client(omni_server): ) -@pytest.fixture(scope="session") -def base64_encoded_video() -> str: - """Base64 encoded video for testing.""" - video = VideoAsset(name="baby_reading", num_frames=4) - with open(video.video_path, "rb") as f: - content = f.read() - return base64.b64encode(content).decode("utf-8") - - def get_system_prompt(): return { "role": "system", @@ -193,71 +98,50 @@ def dummy_messages_from_video_data( ] +def get_prompt(prompt_type="text_only"): + prompts = { + "text_only": "What is the capital of China?", + "mix": "What is recited in the audio? What is in this image? Describe the video briefly.", + } + return prompts.get(prompt_type, prompts["text_only"]) + + +def get_max_batch_size(size_type="few"): + batch_sizes = {"few": 5, "medium": 100, "large": 256} + return batch_sizes.get(size_type, 5) + + +@pytest.mark.skipif(is_rocm(), reason="Test skipped on AMD environment due to known output issues") @pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_video_to_audio_concurrent( - client: openai.OpenAI, - omni_server, - base64_encoded_video: str, -) -> None: - """Test processing video with multiple concurrent completions, generating audio output via OpenAI API.""" - # Create data URL for the base64 encoded video - video_data_url = f"data:video/mp4;base64,{base64_encoded_video}" +def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server) -> None: + """ + Test multi-modal input processing and text/audio output generation via OpenAI API. + Deploy Setting: default yaml + Input Modal: text + audio + video + image + Output Modal: text + audio + Input Setting: stream=True + Datasets: single request + """ - messages = dummy_messages_from_video_data(video_data_url) - - # Test multiple concurrent completions - num_concurrent_requests = 5 - - with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: - # Submit multiple completion requests concurrently - futures = [ - executor.submit( - client.chat.completions.create, - model=omni_server.model, - messages=messages, - ) - for _ in range(num_concurrent_requests) - ] - - # Wait for all requests to complete and collect results - chat_completions = [future.result() for future in concurrent.futures.as_completed(futures)] - - # Verify all completions succeeded - assert len(chat_completions) == num_concurrent_requests - - for chat_completion in chat_completions: - assert len(chat_completion.choices) == 2 # 1 for text output, 1 for audio output - - # Verify text output - text_choice = chat_completion.choices[0] - assert text_choice.finish_reason == "length" - - # Verify we got a response - text_message = text_choice.message - assert text_message.content is not None and len(text_message.content) >= 10 - assert text_message.role == "assistant" - - # Verify audio output - audio_choice = chat_completion.choices[1] - assert audio_choice.finish_reason == "stop" - audio_message = audio_choice.message - - # Check if audio was generated - if hasattr(audio_message, "audio") and audio_message.audio: - assert audio_message.audio.data is not None - assert len(audio_message.audio.data) > 0 - - # Test streaming completion - chat_completion = client.chat.completions.create( - model=omni_server.model, - messages=messages, - stream=True, + # Test single completion + e2e_list = list() + video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}" + image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}" + audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" + messages = dummy_messages_from_mix_data( + system_prompt=get_system_prompt(), + video_data_url=video_data_url, + image_data_url=image_data_url, + audio_data_url=audio_data_url, + content_text=get_prompt("mix"), ) - # Collect text and audio data from stream + # Test single completion + start_time = time.perf_counter() + chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True) + text_content = "" audio_data = None - for chunk in chat_completion: for choice in chunk.choices: if hasattr(choice, "delta"): @@ -268,18 +152,92 @@ def test_video_to_audio_concurrent( modality = getattr(chunk, "modality", None) if modality == "audio" and content: - # Audio chunk - decode base64 content + # Audio chunk - content if audio_data is None: - audio_data = base64.b64decode(content) + audio_data = content else: - audio_data += base64.b64decode(content) + audio_data += content elif modality == "text" and content: # Text chunk - accumulate text content text_content += content if content else "" - # Verify text output - assert text_content is not None and len(text_content) >= 2 + # Verify E2E + current_e2e = time.perf_counter() - start_time + print(f"the request e2e is: {current_e2e}") + # TODO: Verify the E2E latency after confirmation baseline. + e2e_list.append(current_e2e) - # Verify audio output - assert audio_data is not None - assert len(audio_data) > 0 + print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}") + # Verify all completions succeeded + assert audio_data is not None, "No audio output is generated" + + # Verify text output success + assert text_content is not None and len(text_content) >= 2, "No text output is generated" + assert any( + keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"] + ), "The output does not contain any of the keywords." + + # Verify text output same as audio output + audio_content = convert_audio_to_text(audio_data) + print(f"text content is: {text_content}") + print(f"audio content is: {audio_content}") + similarity = cosine_similarity_text(audio_content.lower(), text_content.lower()) + print(f"similarity is: {similarity}") + assert similarity > 0.9, "The audio content is not same as the text" + + +@pytest.mark.skipif(is_rocm(), reason="Test skipped on AMD environment due to known output issues") +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None: + """ + Test text input processing and text/audio output generation via OpenAI API. + Deploy Setting: default yaml + Input Modal: text + Output Modal: text + audio + Datasets: few requests + """ + + num_concurrent_requests = get_max_batch_size() + messages = dummy_messages_from_mix_data(system_prompt=get_system_prompt(), content_text=get_prompt()) + + e2e_list = list() + with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: + # Submit multiple completion requests concurrently + futures = [ + executor.submit(client.chat.completions.create, model=omni_server.model, messages=messages) + for _ in range(num_concurrent_requests) + ] + start_time = time.perf_counter() + # Wait for all requests to complete and collect results + chat_completions = list() + for future in concurrent.futures.as_completed(futures): + chat_completions.append(future.result()) + # Verify E2E + current_e2e = time.perf_counter() - start_time + print(f"the request e2e is: {current_e2e}") + # TODO: Verify the E2E latency after confirmation baseline. + e2e_list.append(current_e2e) + + print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}") + # Verify all completions succeeded + assert len(chat_completions) == num_concurrent_requests, "Not all requests succeeded." + for chat_completion in chat_completions: + # Verify audio output success + audio_message = chat_completion.choices[1].message + audio_data = audio_message.audio.data + assert audio_data is not None, "No audio output is generated" + assert audio_message.audio.expires_at > time.time(), "The generated audio has expired." + + # Verify text output success + text_choice = chat_completion.choices[0] + text_content = text_choice.message.content + assert text_choice.message.content is not None, "No text output is generated" + assert "beijing" in text_choice.message.content.lower(), "The output do not contain keywords." + + # Verify text output same as audio output + audio_content = convert_audio_to_text(audio_data) + print(f"text content is: {text_content}") + print(f"audio content is: {audio_content}") + similarity = cosine_similarity_text(audio_content.lower(), text_content.lower()) + print(f"similarity is: {similarity}") + assert similarity > 0.9, "The audio content is not same as the text" diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index a779993bf..563eaf628 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -111,69 +111,6 @@ def test_text_to_text_001(test_config: tuple[str, str]) -> None: assert "beijing" in text_choice.message.content.lower(), "The output do not contain keywords." -@pytest.mark.parametrize("test_config", test_params) -def test_text_to_text_audio_001(test_config: tuple[str, str]) -> None: - """Test processing text, generating text and audio output via OpenAI API.""" - - model, stage_config_path = test_config - num_concurrent_requests = get_max_batch_size() - stage_config_path = modify_stage_config( - stage_config_path, - { - 0: {"runtime.max_batch_size": num_concurrent_requests}, - 1: {"runtime.max_batch_size": num_concurrent_requests}, - }, - ) - with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server: - messages = dummy_messages_from_mix_data( - system_prompt=get_system_prompt(), content_text="What is the capital of China?" - ) - - # Test single completion - api_client = client(server) - e2e_list = list() - with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: - # Submit multiple completion requests concurrently - futures = [ - executor.submit(api_client.chat.completions.create, model=server.model, messages=messages) - for _ in range(num_concurrent_requests) - ] - start_time = time.perf_counter() - # Wait for all requests to complete and collect results - chat_completions = list() - for future in concurrent.futures.as_completed(futures): - chat_completions.append(future.result()) - # Verify E2E - current_e2e = time.perf_counter() - start_time - print(f"the request e2e is: {current_e2e}") - # TODO: Verify the E2E latency after confirmation baseline. - e2e_list.append(current_e2e) - - print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}") - # Verify all completions succeeded - assert len(chat_completions) == num_concurrent_requests, "Not all requests succeeded." - for chat_completion in chat_completions: - # Verify audio output success - audio_message = chat_completion.choices[1].message - audio_data = audio_message.audio.data - assert audio_data is not None, "No audio output is generated" - assert audio_message.audio.expires_at > time.time(), "The generated audio has expired." - - # Verify text output success - text_choice = chat_completion.choices[0] - text_content = text_choice.message.content - assert text_choice.message.content is not None, "No text output is generated" - assert "beijing" in text_choice.message.content.lower(), "The output do not contain keywords." - - # Verify text output same as audio output - audio_content = convert_audio_to_text(audio_data) - print(f"text content is: {text_content}") - print(f"audio content is: {audio_content}") - assert cosine_similarity_text(audio_content.lower(), text_content.lower()) > 0.9, ( - "The audio content is not same as the text" - ) - - @pytest.mark.parametrize("test_config", test_params) def test_audio_to_text_001(test_config: tuple[str, str]) -> None: """Test processing text, generating text output via OpenAI API.""" diff --git a/tests/e2e/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/stage_configs/qwen3_omni_ci.yaml index 24eeeb8d6..4b5638529 100644 --- a/tests/e2e/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/stage_configs/qwen3_omni_ci.yaml @@ -8,7 +8,7 @@ stage_args: - stage_id: 0 runtime: devices: "0,1" - max_batch_size: 1 + max_batch_size: 5 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration @@ -39,7 +39,7 @@ stage_args: - stage_id: 1 runtime: devices: "1" - max_batch_size: 1 + max_batch_size: 5 engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration From 5d9918fa67525dd3196bf51e3dba5bfe0b125096 Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Fri, 23 Jan 2026 15:16:40 +0800 Subject: [PATCH 07/83] [Misc] Fix error log for the diffusion stage timeout (#915) Signed-off-by: samithuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> Signed-off-by: Hongsheng Liu Co-authored-by: Hongsheng Liu Co-authored-by: Jiangyun Zhu --- vllm_omni/entrypoints/omni.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm_omni/entrypoints/omni.py b/vllm_omni/entrypoints/omni.py index a5f5179e2..3d918aea7 100644 --- a/vllm_omni/entrypoints/omni.py +++ b/vllm_omni/entrypoints/omni.py @@ -345,6 +345,7 @@ class OmniBase: ) suggestions = [ + f"Ignore this warning if the model weight download / load from disk time is longer than {timeout}s.", "Verify GPU/device assignment in config (runtime.devices) is correct.", "Check GPU/host memory availability; reduce model or batch size if needed.", "Check model weights path and network reachability (if loading remotely).", @@ -353,7 +354,7 @@ class OmniBase: formatted_suggestions = "\n".join(f" {i + 1}) {msg}" for i, msg in enumerate(suggestions)) - logger.error(f"[{self._name}] Stage initialization failed. Troubleshooting Steps:\n{formatted_suggestions}") + logger.warning(f"[{self._name}] Stage initialization timeout. Troubleshooting Steps:\n{formatted_suggestions}") def start_profile(self, stages: list[int] | None = None) -> None: """Start profiling for specified stages. From 21ee1ff6e6e31897acb10518ea631be4fa5529b8 Mon Sep 17 00:00:00 2001 From: Baoyuan Qi Date: Fri, 23 Jan 2026 17:37:53 +0800 Subject: [PATCH 08/83] [bugfix] qwen3-tts check_model_inputs (#924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 齐保元 Co-authored-by: shijin zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: ning ding --- .../qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py index fb09f9b88..11c715f5f 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py +++ b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py @@ -495,7 +495,7 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr # Initialize weights and apply final processing self.post_init() - @check_model_inputs() + @check_model_inputs @auto_docstring def forward( self, From 28a6597c4ccfb39c60ce675f0c0da367ef7904f8 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Fri, 23 Jan 2026 19:32:06 +0800 Subject: [PATCH 09/83] [Misc] Fix t2i online serving example (#928) Signed-off-by: zjy0516 --- .../text_to_image/openai_chat_client.py | 58 +++++++++---------- .../text_to_image/run_curl_text_to_image.sh | 37 ++---------- 2 files changed, 33 insertions(+), 62 deletions(-) diff --git a/examples/online_serving/text_to_image/openai_chat_client.py b/examples/online_serving/text_to_image/openai_chat_client.py index 39fa7dc22..7beac1acc 100644 --- a/examples/online_serving/text_to_image/openai_chat_client.py +++ b/examples/online_serving/text_to_image/openai_chat_client.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Qwen-Image OpenAI-compatible chat client for image generation. +Qwen-Image OpenAI-compatible image generation client. Usage: python openai_chat_client.py --prompt "A beautiful landscape" --output output.png @@ -25,14 +25,14 @@ def generate_image( negative_prompt: str | None = None, num_outputs_per_prompt: int = 1, ) -> bytes | None: - """Generate an image using the chat completions API. + """Generate an image using the images generation API. Args: prompt: Text description of the image server_url: Server URL height: Image height in pixels width: Image width in pixels - steps: Number of inference steps + steps: Number of diffusion steps true_cfg_scale: Qwen-Image CFG scale seed: Random seed negative_prompt: Negative prompt @@ -41,33 +41,31 @@ def generate_image( Returns: Image bytes or None if failed """ - messages = [{"role": "user", "content": prompt}] + payload: dict[str, object] = { + "prompt": prompt, + "response_format": "b64_json", + "n": num_outputs_per_prompt, + } + + if width is not None and height is not None: + payload["size"] = f"{width}x{height}" + elif width is not None: + payload["size"] = f"{width}x{width}" + elif height is not None: + payload["size"] = f"{height}x{height}" - # Build extra_body with generation parameters - extra_body = {} - if height is not None: - extra_body["height"] = height - if width is not None: - extra_body["width"] = width if steps is not None: - extra_body["num_inference_steps"] = steps + payload["num_inference_steps"] = steps if true_cfg_scale is not None: - extra_body["true_cfg_scale"] = true_cfg_scale - if seed is not None: - extra_body["seed"] = seed + payload["true_cfg_scale"] = true_cfg_scale if negative_prompt: - extra_body["negative_prompt"] = negative_prompt - extra_body["num_outputs_per_prompt"] = num_outputs_per_prompt + payload["negative_prompt"] = negative_prompt + if seed is not None: + payload["seed"] = seed - # Build request payload - payload = {"messages": messages} - if extra_body: - payload["extra_body"] = extra_body - - # Send request try: response = requests.post( - f"{server_url}/v1/chat/completions", + f"{server_url}/v1/images/generations", headers={"Content-Type": "application/json"}, json=payload, timeout=300, @@ -75,15 +73,13 @@ def generate_image( response.raise_for_status() data = response.json() - # Extract image from response - content = data["choices"][0]["message"]["content"] - if isinstance(content, list) and len(content) > 0: - image_url = content[0].get("image_url", {}).get("url", "") - if image_url.startswith("data:image"): - _, b64_data = image_url.split(",", 1) - return base64.b64decode(b64_data) + items = data.get("data") + if isinstance(items, list) and items: + first = items[0].get("b64_json") if isinstance(items[0], dict) else None + if isinstance(first, str): + return base64.b64decode(first) - print(f"Unexpected response format: {content}") + print(f"Unexpected response format: {data}") return None except Exception as e: diff --git a/examples/online_serving/text_to_image/run_curl_text_to_image.sh b/examples/online_serving/text_to_image/run_curl_text_to_image.sh index 2f559d563..151df8948 100755 --- a/examples/online_serving/text_to_image/run_curl_text_to_image.sh +++ b/examples/online_serving/text_to_image/run_curl_text_to_image.sh @@ -1,35 +1,10 @@ #!/bin/bash # Qwen-Image text-to-image curl example -SERVER="${SERVER:-http://localhost:8091}" -PROMPT="${PROMPT:-a good boy in the ocean}" -CURRENT_TIME=$(date +%Y%m%d%H%M%S) -OUTPUT="${OUTPUT:-qwen_image_output_${CURRENT_TIME}.png}" - -echo "Generating image..." -echo "Prompt: $PROMPT" -echo "Output: $OUTPUT" - -curl -s "$SERVER/v1/chat/completions" \ +curl -X POST http://localhost:8091/v1/images/generations \ -H "Content-Type: application/json" \ - -d "{ - \"messages\": [ - {\"role\": \"user\", \"content\": \"$PROMPT\"} - ], - \"extra_body\": { - \"height\": 1024, - \"width\": 1024, - \"num_inference_steps\": 100, - \"true_cfg_scale\": 4.0, - \"seed\": 42, - \"num_outputs_per_prompt\": 1 - } - }" | jq -r '.choices[0].message.content[0].image_url.url' | sed 's/^data:image[^,]*,\s*//' | base64 -d > "$OUTPUT" - -if [ -f "$OUTPUT" ]; then - echo "Image saved to: $OUTPUT" - echo "Size: $(du -h "$OUTPUT" | cut -f1)" -else - echo "Failed to generate image" - exit 1 -fi + -d '{ + "prompt": "a dragon laying over the spine of the Green Mountains of Vermont", + "size": "1024x1024", + "seed": 42 + }' | jq -r '.data[0].b64_json' | base64 -d > dragon.png From 3d9cd16df741c6de4e74ea9bd2511de661f6c1c0 Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Fri, 23 Jan 2026 22:43:00 +0800 Subject: [PATCH 10/83] [examples] add --enable-cpu-offload args (#930) Signed-off-by: David Chen <530634352@qq.com> --- examples/offline_inference/image_to_image/image_edit.py | 6 ++++++ examples/offline_inference/image_to_image/image_to_image.md | 1 + examples/offline_inference/image_to_video/README.md | 1 + examples/offline_inference/image_to_video/image_to_video.py | 6 ++++++ examples/offline_inference/text_to_image/README.md | 1 + examples/offline_inference/text_to_image/text_to_image.py | 6 ++++++ examples/offline_inference/text_to_video/text_to_video.md | 1 + examples/offline_inference/text_to_video/text_to_video.py | 6 ++++++ 8 files changed, 28 insertions(+) diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index 2f1a8166d..405a0cafa 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -279,6 +279,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Disable torch.compile and force eager execution.", ) + parser.add_argument( + "--enable-cpu-offload", + action="store_true", + help="Enable CPU offloading for diffusion models.", + ) return parser.parse_args() @@ -344,6 +349,7 @@ def main(): cache_config=cache_config, parallel_config=parallel_config, enforce_eager=args.enforce_eager, + enable_cpu_offload=args.enable_cpu_offload, ) print("Pipeline loaded") diff --git a/examples/offline_inference/image_to_image/image_to_image.md b/examples/offline_inference/image_to_image/image_to_image.md index 87bb8b68d..d624b4684 100644 --- a/examples/offline_inference/image_to_image/image_to_image.md +++ b/examples/offline_inference/image_to_image/image_to_image.md @@ -47,3 +47,4 @@ Key arguments: - `--guidance_scale`: guidance scale for guidance-distilled models (default: 1.0, disabled). Unlike classifier-free guidance (--cfg_scale), guidance-distilled models take the guidance scale directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models. - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--output`: path to save the generated PNG. +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. diff --git a/examples/offline_inference/image_to_video/README.md b/examples/offline_inference/image_to_video/README.md index b2c141683..afebb1963 100644 --- a/examples/offline_inference/image_to_video/README.md +++ b/examples/offline_inference/image_to_video/README.md @@ -54,3 +54,4 @@ Key arguments: - `--num_inference_steps`: Number of denoising steps (default 50). - `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: Path to save the generated video. +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index f40eebaef..2104ec0fd 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -58,6 +58,11 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--output", type=str, default="i2v_output.mp4", help="Path to save the video (mp4).") parser.add_argument("--fps", type=int, default=16, help="Frames per second for the output video.") + parser.add_argument( + "--enable-cpu-offload", + action="store_true", + help="Enable CPU offloading for diffusion models.", + ) return parser.parse_args() @@ -105,6 +110,7 @@ def main(): vae_use_tiling=vae_use_tiling, boundary_ratio=args.boundary_ratio, flow_shift=args.flow_shift, + enable_cpu_offload=args.enable_cpu_offload, ) if profiler_enabled: diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index daf014041..0f4351ac2 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -61,6 +61,7 @@ Key arguments: - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--height/--width`: output resolution (defaults 1024x1024). - `--output`: path to save the generated PNG. +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. > ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes. diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index b07dbb27a..428c2437e 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -96,6 +96,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Disable torch.compile and force eager execution.", ) + parser.add_argument( + "--enable-cpu-offload", + action="store_true", + help="Enable CPU offloading for diffusion models.", + ) parser.add_argument( "--tensor_parallel_size", type=int, @@ -162,6 +167,7 @@ def main(): cache_config=cache_config, parallel_config=parallel_config, enforce_eager=args.enforce_eager, + enable_cpu_offload=args.enable_cpu_offload, ) if profiler_enabled: diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index da622dcd2..f0ce5495a 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -28,3 +28,4 @@ Key arguments: - `--boundary_ratio`: Boundary split ratio for low/high DiT. - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: path to save the generated video. +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index f50692ce6..066714c54 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -35,6 +35,11 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--output", type=str, default="wan22_output.mp4", help="Path to save the video (mp4).") parser.add_argument("--fps", type=int, default=24, help="Frames per second for the output video.") + parser.add_argument( + "--enable-cpu-offload", + action="store_true", + help="Enable CPU offloading for diffusion models.", + ) return parser.parse_args() @@ -56,6 +61,7 @@ def main(): vae_use_tiling=vae_use_tiling, boundary_ratio=args.boundary_ratio, flow_shift=args.flow_shift, + enable_cpu_offload=args.enable_cpu_offload, ) if profiler_enabled: From f9c69a8c401ff00798bbf405c905b14ab6662a55 Mon Sep 17 00:00:00 2001 From: YinHanke Date: Sat, 24 Jan 2026 17:27:58 +0800 Subject: [PATCH 11/83] [BugFix] Standardize StableAudio audio output (#842) Signed-off-by: LudovicoYIN Co-authored-by: Hongsheng Liu --- .../text_to_audio/text_to_audio.py | 17 +++- .../test_stable_audio_model.py | 9 +-- vllm_omni/diffusion/diffusion_engine.py | 79 +++++++++++++------ vllm_omni/diffusion/models/interface.py | 5 ++ .../stable_audio/pipeline_stable_audio.py | 3 +- vllm_omni/outputs.py | 7 +- 6 files changed, 90 insertions(+), 30 deletions(-) diff --git a/examples/offline_inference/text_to_audio/text_to_audio.py b/examples/offline_inference/text_to_audio/text_to_audio.py index 0b172a077..ca1a455d1 100644 --- a/examples/offline_inference/text_to_audio/text_to_audio.py +++ b/examples/offline_inference/text_to_audio/text_to_audio.py @@ -142,7 +142,7 @@ def main(): generation_start = time.perf_counter() # Generate audio - audio = omni.generate( + outputs = omni.generate( args.prompt, negative_prompt=args.negative_prompt, generator=generator, @@ -166,6 +166,21 @@ def main(): suffix = output_path.suffix or ".wav" stem = output_path.stem or "stable_audio_output" + # Extract audio from omni.generate() outputs + if not outputs: + raise ValueError("No output generated from omni.generate()") + + output = outputs[0] + if not hasattr(output, "request_output") or not output.request_output: + raise ValueError("No request_output found in OmniRequestOutput") + request_output = output.request_output[0] + if not hasattr(request_output, "multimodal_output"): + raise ValueError("No multimodal_output found in request_output") + + audio = request_output.multimodal_output.get("audio") + if audio is None: + raise ValueError("No audio output found in request_output") + # Handle different output formats if isinstance(audio, torch.Tensor): audio = audio.cpu().float().numpy() diff --git a/tests/e2e/offline_inference/test_stable_audio_model.py b/tests/e2e/offline_inference/test_stable_audio_model.py index 648562573..729b501e4 100644 --- a/tests/e2e/offline_inference/test_stable_audio_model.py +++ b/tests/e2e/offline_inference/test_stable_audio_model.py @@ -44,15 +44,14 @@ def test_stable_audio_model(model_name: str): # Extract audio from OmniRequestOutput assert outputs is not None first_output = outputs[0] - assert first_output.final_output_type == "image" # Generic output type + assert first_output.final_output_type == "image" assert hasattr(first_output, "request_output") and first_output.request_output req_out = first_output.request_output[0] assert isinstance(req_out, OmniRequestOutput) - assert hasattr(req_out, "images") and len(req_out.images) >= 1 - - # For stable audio, the "images" field contains audio numpy arrays - audio = req_out.images[0] + assert req_out.final_output_type == "audio" + assert hasattr(req_out, "multimodal_output") and req_out.multimodal_output + audio = req_out.multimodal_output.get("audio") assert isinstance(audio, np.ndarray) # audio shape: (batch, channels, samples) # For stable-audio-open-1.0: sample_rate=44100, so 2 seconds = 88200 samples diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 731ed425e..e85ae0d76 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -29,6 +29,13 @@ def supports_image_input(model_class_name: str) -> bool: return bool(getattr(model_cls, "support_image_input", False)) +def supports_audio_output(model_class_name: str) -> bool: + model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name) + if model_cls is None: + return False + return bool(getattr(model_cls, "support_audio_output", False)) + + class DiffusionEngine: """The diffusion engine for vLLM-Omni diffusion models.""" @@ -86,14 +93,14 @@ class DiffusionEngine: return None postprocess_start_time = time.time() - images = self.post_process_func(output.output) if self.post_process_func is not None else output.output + outputs = self.post_process_func(output.output) if self.post_process_func is not None else output.output postprocess_time = time.time() - postprocess_start_time logger.info(f"Post-processing completed in {postprocess_time:.4f} seconds") # Convert to OmniRequestOutput format - # Ensure images is a list - if not isinstance(images, list): - images = [images] if images is not None else [] + # Ensure outputs is a list + if not isinstance(outputs, list): + outputs = [outputs] if outputs is not None else [] # Handle single request or multiple requests if len(requests) == 1: @@ -108,18 +115,30 @@ class DiffusionEngine: if output.trajectory_timesteps is not None: metrics["trajectory_timesteps"] = output.trajectory_timesteps - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=images, - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, - ) + if supports_audio_output(self.od_config.model_class_name): + audio_payload = outputs[0] if len(outputs) == 1 else outputs + return OmniRequestOutput.from_diffusion( + request_id=request_id, + images=[], + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + multimodal_output={"audio": audio_payload}, + final_output_type="audio", + ) + else: + return OmniRequestOutput.from_diffusion( + request_id=request_id, + images=outputs, + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + ) else: # Multiple requests: return list of OmniRequestOutput # Split images based on num_outputs_per_prompt for each request results = [] - image_idx = 0 + output_idx = 0 for request in requests: request_id = request.request_id or "" @@ -129,22 +148,38 @@ class DiffusionEngine: # Get images for this request num_outputs = request.num_outputs_per_prompt - request_images = images[image_idx : image_idx + num_outputs] if image_idx < len(images) else [] - image_idx += num_outputs + request_outputs = ( + outputs[output_idx : output_idx + num_outputs] if output_idx < len(outputs) else [] + ) + output_idx += num_outputs metrics = {} if output.trajectory_timesteps is not None: metrics["trajectory_timesteps"] = output.trajectory_timesteps - results.append( - OmniRequestOutput.from_diffusion( - request_id=request_id, - images=request_images, - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, + if supports_audio_output(self.od_config.model_class_name): + audio_payload = request_outputs[0] if len(request_outputs) == 1 else request_outputs + results.append( + OmniRequestOutput.from_diffusion( + request_id=request_id, + images=[], + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + multimodal_output={"audio": audio_payload}, + final_output_type="audio", + ) + ) + else: + results.append( + OmniRequestOutput.from_diffusion( + request_id=request_id, + images=request_outputs, + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + ) ) - ) return results except Exception as e: diff --git a/vllm_omni/diffusion/models/interface.py b/vllm_omni/diffusion/models/interface.py index 7c4cb33e6..8e8ecbb77 100644 --- a/vllm_omni/diffusion/models/interface.py +++ b/vllm_omni/diffusion/models/interface.py @@ -10,3 +10,8 @@ from typing import ( @runtime_checkable class SupportImageInput(Protocol): support_image_input: ClassVar[bool] = True + + +@runtime_checkable +class SupportAudioOutput(Protocol): + support_audio_output: ClassVar[bool] = True diff --git a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py index 2639f55ef..14516a9bb 100644 --- a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py +++ b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py @@ -27,6 +27,7 @@ from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.models.interface import SupportAudioOutput from vllm_omni.diffusion.models.stable_audio.stable_audio_transformer import StableAudioDiTModel from vllm_omni.diffusion.request import OmniDiffusionRequest @@ -57,7 +58,7 @@ def get_stable_audio_post_process_func( return post_process_func -class StableAudioPipeline(nn.Module): +class StableAudioPipeline(nn.Module, SupportAudioOutput): """ Pipeline for text-to-audio generation using Stable Audio Open. diff --git a/vllm_omni/outputs.py b/vllm_omni/outputs.py index 4e0583d77..24b168287 100644 --- a/vllm_omni/outputs.py +++ b/vllm_omni/outputs.py @@ -57,6 +57,7 @@ class OmniRequestOutput: prompt: str | None = None latents: torch.Tensor | None = None metrics: dict[str, Any] = field(default_factory=dict) + multimodal_output: dict[str, Any] = field(default_factory=dict) @classmethod def from_pipeline( @@ -91,6 +92,8 @@ class OmniRequestOutput: prompt: str | None = None, metrics: dict[str, Any] | None = None, latents: torch.Tensor | None = None, + multimodal_output: dict[str, Any] | None = None, + final_output_type: str = "image", ) -> "OmniRequestOutput": """Create output from diffusion model. @@ -106,11 +109,12 @@ class OmniRequestOutput: """ return cls( request_id=request_id, - final_output_type="image", + final_output_type=final_output_type, images=images, prompt=prompt, latents=latents, metrics=metrics or {}, + multimodal_output=multimodal_output or {}, finished=True, ) @@ -171,6 +175,7 @@ class OmniRequestOutput: f"prompt={self.prompt!r}", f"latents={self.latents}", f"metrics={self.metrics}", + f"multimodal_output={self.multimodal_output}", ] return f"OmniRequestOutput({', '.join(parts)})" From 06c9d9b463b4aeb936dd34938b2d87347f645998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Sun, 25 Jan 2026 14:52:27 +0800 Subject: [PATCH 12/83] [Bagel] Support TeaCache (#848) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: princepride Signed-off-by: 汪志鹏 --- docs/user_guide/diffusion_acceleration.md | 2 +- vllm_omni/diffusion/cache/teacache/backend.py | 91 +++++++++++----- vllm_omni/diffusion/cache/teacache/config.py | 3 + .../diffusion/cache/teacache/extractors.py | 102 ++++++++++++++++++ vllm_omni/diffusion/cache/teacache/hook.py | 2 +- 5 files changed, 172 insertions(+), 28 deletions(-) diff --git a/docs/user_guide/diffusion_acceleration.md b/docs/user_guide/diffusion_acceleration.md index 4572b7115..42202d8d7 100644 --- a/docs/user_guide/diffusion_acceleration.md +++ b/docs/user_guide/diffusion_acceleration.md @@ -49,7 +49,7 @@ The following table shows which models are currently supported by each accelerat | **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ❌ | ✅ | ✅ | ✅ | ✅ | | **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ✅ | ✅ | ❌ | ❌ | ❌ | | **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ✅ | ❌ | ❌ | ❌ | -| **Bagel** | `ByteDance-Seed/BAGEL-7B-MoT` | ❌ | ✅ | ❌ | ❌ | ❌ | +| **Bagel** | `ByteDance-Seed/BAGEL-7B-MoT` | ✅ | ✅ | ❌ | ❌ | ❌ | ### VideoGen diff --git a/vllm_omni/diffusion/cache/teacache/backend.py b/vllm_omni/diffusion/cache/teacache/backend.py index 9db153cc4..bf328d43d 100644 --- a/vllm_omni/diffusion/cache/teacache/backend.py +++ b/vllm_omni/diffusion/cache/teacache/backend.py @@ -15,10 +15,42 @@ from vllm.logger import init_logger from vllm_omni.diffusion.cache.base import CacheBackend from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache_hook +from vllm_omni.diffusion.data import DiffusionCacheConfig logger = init_logger(__name__) +def enable_bagel_teacache(pipeline: Any, config: DiffusionCacheConfig) -> None: + """ + Enable TeaCache for Bagel model. + """ + teacache_config = TeaCacheConfig( + transformer_type="Bagel", + rel_l1_thresh=config.rel_l1_thresh, + coefficients=config.coefficients, + ) + transformer = pipeline.bagel + original_forward_flow = transformer._forward_flow + + import types + + def forward_alias(self, *args, **kwargs): + return original_forward_flow(*args, **kwargs) + + transformer.forward = types.MethodType(forward_alias, transformer) + apply_teacache_hook(transformer, teacache_config) + transformer._forward_flow = transformer.forward + pipeline.transformer = transformer + + logger.info( + f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, " + f"transformer_class={teacache_config.transformer_type}" + ) + + +CUSTOM_TEACACHE_ENABLERS = {"BagelPipeline": enable_bagel_teacache} + + class TeaCacheBackend(CacheBackend): """ TeaCache implementation using hooks. @@ -51,38 +83,45 @@ class TeaCacheBackend(CacheBackend): - transformer: pipeline.transformer - transformer_type: pipeline.transformer.__class__.__name__ """ - # Extract transformer and transformer_type from pipeline - transformer = pipeline.transformer - transformer_type = transformer.__class__.__name__ + # Helper to get pipeline class name + pipeline_type = pipeline.__class__.__name__ - # Create TeaCacheConfig from DiffusionCacheConfig with transformer_type - # Access parameters via attribute access: config.rel_l1_thresh - # rel_l1_thresh already has a default value of 0.2 in DiffusionCacheConfig - try: - teacache_config = TeaCacheConfig( - transformer_type=transformer_type, - rel_l1_thresh=self.config.rel_l1_thresh, - coefficients=self.config.coefficients, - ) - except Exception as e: - logger.error(f"Failed to create TeaCacheConfig: {e}") - raise ValueError( - f"Invalid TeaCache configuration: {e}. " - f"Expected keys: rel_l1_thresh, coefficients (optional). " - f"transformer_type is automatically extracted from pipeline.transformer.__class__.__name__." - ) + # Check for pipeline-level custom enablers + if pipeline_type in CUSTOM_TEACACHE_ENABLERS: + logger.info(f"Using custom TeaCache enabler for model: {pipeline_type}") + CUSTOM_TEACACHE_ENABLERS[pipeline_type](pipeline, self.config) + else: + transformer = pipeline.transformer + transformer_type = transformer.__class__.__name__ - # Apply hook to transformer - apply_teacache_hook(transformer, teacache_config) + # Create TeaCacheConfig from DiffusionCacheConfig with transformer_type + # Access parameters via attribute access: config.rel_l1_thresh + # rel_l1_thresh already has a default value of 0.2 in DiffusionCacheConfig + try: + teacache_config = TeaCacheConfig( + transformer_type=transformer_type, + rel_l1_thresh=self.config.rel_l1_thresh, + coefficients=self.config.coefficients, + ) + except Exception as e: + logger.error(f"Failed to create TeaCacheConfig: {e}") + raise ValueError( + f"Invalid TeaCache configuration: {e}. " + f"Expected keys: rel_l1_thresh, coefficients (optional). " + f"transformer_type is automatically extracted from pipeline.transformer.__class__.__name__." + ) + + # Apply hook to transformer + apply_teacache_hook(transformer, teacache_config) + + logger.info( + f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, " + f"transformer_class={teacache_config.transformer_type}" + ) # Mark as enabled self.enabled = True - logger.info( - f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, " - f"transformer_class={teacache_config.transformer_type}" - ) - def refresh(self, pipeline: Any, num_inference_steps: int, verbose: bool = True) -> None: """ Refresh TeaCache state for new generation. diff --git a/vllm_omni/diffusion/cache/teacache/config.py b/vllm_omni/diffusion/cache/teacache/config.py index bb5e7cb27..30b1745f4 100644 --- a/vllm_omni/diffusion/cache/teacache/config.py +++ b/vllm_omni/diffusion/cache/teacache/config.py @@ -25,6 +25,9 @@ _MODEL_COEFFICIENTS = { 3.20000000e00, -2.00000000e-02, ], + # Bagel transformer coefficients + # Using Qwen's coefficients as reasonable default given shared architecture + "Bagel": [1.33313129e06, -1.68644226e05, 7.95050740e03, -1.63747873e02, 1.26352397e00], # Z-Image transformer coefficients # Copied from Qwen-Image, need to be tuned specifically for Z-Image in future "ZImageTransformer2DModel": [ diff --git a/vllm_omni/diffusion/cache/teacache/extractors.py b/vllm_omni/diffusion/cache/teacache/extractors.py index 5c35e0c8c..01d453061 100644 --- a/vllm_omni/diffusion/cache/teacache/extractors.py +++ b/vllm_omni/diffusion/cache/teacache/extractors.py @@ -254,6 +254,107 @@ def extract_qwen_context( ) +def extract_bagel_context( + module: nn.Module, + x_t: torch.Tensor, + timestep: torch.Tensor | float | int, + packed_vae_token_indexes: torch.LongTensor, + packed_vae_position_ids: torch.LongTensor, + packed_text_ids: torch.LongTensor, + packed_text_indexes: torch.LongTensor, + packed_indexes: torch.LongTensor, + packed_position_ids: torch.LongTensor, + packed_seqlens: torch.IntTensor, + key_values_lens: torch.IntTensor, + past_key_values: Any, + packed_key_value_indexes: torch.LongTensor, + **kwargs: Any, +) -> CacheContext: + """ + Extract cache context for Bagel model. + + Args: + module: Bagel instance + x_t: Latent image input + timestep: Current timestep + packed_vae_token_indexes: Indexes for VAE tokens in packed sequence + packed_vae_position_ids: Position IDs for VAE tokens + packed_text_ids: Text token IDs + packed_text_indexes: Indexes for text tokens in packed sequence + packed_indexes: Global indexes + packed_position_ids: Global position IDs + packed_seqlens: Sequence lengths + key_values_lens: KV cache lengths + past_key_values: KV cache + packed_key_value_indexes: KV cache indexes + **kwargs: Additional keyword arguments + + Returns: + CacheContext with all information needed for generic caching + """ + + # 1. Embed text + packed_text_embedding = module.language_model.model.embed_tokens(packed_text_ids) + packed_sequence = packed_text_embedding.new_zeros((sum(packed_seqlens), module.hidden_size)) + packed_sequence[packed_text_indexes] = packed_text_embedding + + # 2. Embed timestep + if not isinstance(timestep, torch.Tensor): + timestep = torch.tensor([timestep], device=x_t.device) + if timestep.dim() == 0: + timestep = timestep.unsqueeze(0) + + # 3. Embed image (x_t) + packed_pos_embed = module.latent_pos_embed(packed_vae_position_ids) + packed_timestep_embeds = module.time_embedder(timestep) + + x_t_emb = module.vae2llm(x_t) + packed_timestep_embeds + packed_pos_embed + if x_t_emb.dtype != packed_sequence.dtype: + x_t_emb = x_t_emb.to(packed_sequence.dtype) + + packed_sequence[packed_vae_token_indexes] = x_t_emb + + # Use the full packed sequence as modulated input to match hidden_states size + modulated_input = packed_sequence + + def run_transformer_blocks(): + extra_inputs = {} + if module.use_moe: + extra_inputs = { + "mode": "gen", + "packed_vae_token_indexes": packed_vae_token_indexes, + "packed_text_indexes": packed_text_indexes, + } + + output = module.language_model.forward( + packed_query_sequence=packed_sequence, + query_lens=packed_seqlens, + packed_query_position_ids=packed_position_ids, + packed_query_indexes=packed_indexes, + past_key_values=past_key_values, + key_values_lens=key_values_lens, + packed_key_value_indexes=packed_key_value_indexes, + update_past_key_values=False, + is_causal=False, + **extra_inputs, + ) + return (output.packed_query_sequence,) + + def postprocess(h): + v_t = module.llm2vae(h) + v_t = v_t[packed_vae_token_indexes] + return v_t + + return CacheContext( + modulated_input=modulated_input, + hidden_states=packed_sequence, # Use full packed sequence + encoder_hidden_states=None, + temb=packed_timestep_embeds, # Approximate + run_transformer_blocks=run_transformer_blocks, + postprocess=postprocess, + ) + + def extract_zimage_context( module: nn.Module, x: list[torch.Tensor], @@ -448,6 +549,7 @@ def extract_zimage_context( # on the transformer module and multiple pipelines can share the same transformer. EXTRACTOR_REGISTRY: dict[str, Callable] = { "QwenImageTransformer2DModel": extract_qwen_context, + "Bagel": extract_bagel_context, "ZImageTransformer2DModel": extract_zimage_context, # Future models: # "FluxTransformer2DModel": extract_flux_context, diff --git a/vllm_omni/diffusion/cache/teacache/hook.py b/vllm_omni/diffusion/cache/teacache/hook.py index d896473b4..65f764c43 100644 --- a/vllm_omni/diffusion/cache/teacache/hook.py +++ b/vllm_omni/diffusion/cache/teacache/hook.py @@ -122,7 +122,7 @@ class TeaCacheHook(ModelHook): # - cfg_rank 0: positive branch # - cfg_rank > 0: negative branch # Without CFG-parallel, branches alternate within a single rank - if module.do_true_cfg: + if getattr(module, "do_true_cfg", False): cfg_parallel_size = get_classifier_free_guidance_world_size() if cfg_parallel_size > 1: cfg_rank = get_classifier_free_guidance_rank() From 7f0825832ae2e6ac758cc96fea7d732c650dcaa9 Mon Sep 17 00:00:00 2001 From: Ding Zuhao Date: Sun, 25 Jan 2026 18:42:02 +0800 Subject: [PATCH 13/83] [Doc] Add Bagel model support to TeaCache documentation (#943) Signed-off-by: Ding Zuhao --- docs/user_guide/diffusion/teacache.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user_guide/diffusion/teacache.md b/docs/user_guide/diffusion/teacache.md index 8331c60f8..3fe961405 100644 --- a/docs/user_guide/diffusion/teacache.md +++ b/docs/user_guide/diffusion/teacache.md @@ -112,6 +112,7 @@ th { | `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | +| `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ### VideoGen From 578ed1967eb8596bf4993bcaa84735cb35ca39f3 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Sun, 25 Jan 2026 23:04:11 +0800 Subject: [PATCH 14/83] [Bugfix] raise error in diffusion engine and fix offload test (#933) Signed-off-by: zjy0516 Co-authored-by: Hongsheng Liu --- .../test_diffusion_cpu_offload.py | 56 ++--- vllm_omni/diffusion/diffusion_engine.py | 208 +++++++++--------- vllm_omni/diffusion/scheduler.py | 3 + .../diffusion/worker/gpu_diffusion_worker.py | 2 +- 4 files changed, 136 insertions(+), 133 deletions(-) diff --git a/tests/e2e/offline_inference/test_diffusion_cpu_offload.py b/tests/e2e/offline_inference/test_diffusion_cpu_offload.py index cd22b34e0..5b43400e8 100644 --- a/tests/e2e/offline_inference/test_diffusion_cpu_offload.py +++ b/tests/e2e/offline_inference/test_diffusion_cpu_offload.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest import torch +from vllm.distributed.parallel_state import cleanup_dist_env_and_memory from tests.utils import GPUMemoryMonitor from vllm_omni.utils.platform_utils import is_npu, is_rocm @@ -17,34 +18,39 @@ from vllm_omni import Omni models = ["riverclouds/qwen_image_random"] +def inference(model_name: str, offload: bool = True): + torch.cuda.empty_cache() + device_index = torch.cuda.current_device() + monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02) + monitor.start() + m = Omni(model=model_name, enable_cpu_offload=offload) + torch.cuda.reset_peak_memory_stats(device=device_index) + height = 256 + width = 256 + + m.generate( + "a photo of a cat sitting on a laptop keyboard", + height=height, + width=width, + num_inference_steps=9, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + ) + peak = monitor.peak_used_mb + monitor.stop() + + return peak + + @pytest.mark.skipif(is_npu() or is_rocm(), reason="Hardware not supported") @pytest.mark.parametrize("model_name", models) def test_cpu_offload_diffusion_model(model_name: str): - def inference(offload: bool = True): - torch.cuda.empty_cache() - device_index = torch.cuda.current_device() - monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02) - monitor.start() - m = Omni(model=model_name, enable_cpu_offload=offload) - torch.cuda.reset_peak_memory_stats(device=device_index) - height = 256 - width = 256 - - m.generate( - "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=9, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - ) - peak = monitor.peak_used_mb - monitor.stop() - - return peak - - offload_peak_memory = inference(offload=True) - no_offload_peak_memory = inference(offload=False) + try: + no_offload_peak_memory = inference(model_name, offload=False) + cleanup_dist_env_and_memory() + offload_peak_memory = inference(model_name, offload=True) + except Exception: + pytest.fail("Inference failed") print(f"Offload peak memory: {offload_peak_memory} MB") print(f"No offload peak memory: {no_offload_peak_memory} MB") assert offload_peak_memory + 2500 < no_offload_peak_memory, ( diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index e85ae0d76..2e00f41b9 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -61,130 +61,124 @@ class DiffusionEngine: raise e def step(self, requests: list[OmniDiffusionRequest]): - try: - # Apply pre-processing if available - if self.pre_process_func is not None: - preprocess_start_time = time.time() - requests = self.pre_process_func(requests) - preprocess_time = time.time() - preprocess_start_time - logger.info(f"Pre-processing completed in {preprocess_time:.4f} seconds") + # Apply pre-processing if available + if self.pre_process_func is not None: + preprocess_start_time = time.time() + requests = self.pre_process_func(requests) + preprocess_time = time.time() - preprocess_start_time + logger.info(f"Pre-processing completed in {preprocess_time:.4f} seconds") - output = self.add_req_and_wait_for_response(requests) - if output.error: - raise Exception(f"{output.error}") - logger.info("Generation completed successfully.") + output = self.add_req_and_wait_for_response(requests) + if output.error: + raise Exception(f"{output.error}") + logger.info("Generation completed successfully.") - if output.output is None: - logger.warning("Output is None, returning empty OmniRequestOutput") - # Return empty output for the first request - if len(requests) > 0: - request = requests[0] - request_id = request.request_id or "" - prompt = request.prompt - if isinstance(prompt, list): - prompt = prompt[0] if prompt else None - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=[], - prompt=prompt, - metrics={}, - latents=None, - ) - return None - - postprocess_start_time = time.time() - outputs = self.post_process_func(output.output) if self.post_process_func is not None else output.output - postprocess_time = time.time() - postprocess_start_time - logger.info(f"Post-processing completed in {postprocess_time:.4f} seconds") - - # Convert to OmniRequestOutput format - # Ensure outputs is a list - if not isinstance(outputs, list): - outputs = [outputs] if outputs is not None else [] - - # Handle single request or multiple requests - if len(requests) == 1: - # Single request: return single OmniRequestOutput + if output.output is None: + logger.warning("Output is None, returning empty OmniRequestOutput") + # Return empty output for the first request + if len(requests) > 0: request = requests[0] request_id = request.request_id or "" prompt = request.prompt if isinstance(prompt, list): prompt = prompt[0] if prompt else None + return OmniRequestOutput.from_diffusion( + request_id=request_id, + images=[], + prompt=prompt, + metrics={}, + latents=None, + ) + return None + + postprocess_start_time = time.time() + outputs = self.post_process_func(output.output) if self.post_process_func is not None else output.output + postprocess_time = time.time() - postprocess_start_time + logger.info(f"Post-processing completed in {postprocess_time:.4f} seconds") + + # Convert to OmniRequestOutput format + # Ensure outputs is a list + if not isinstance(outputs, list): + outputs = [outputs] if outputs is not None else [] + + # Handle single request or multiple requests + if len(requests) == 1: + # Single request: return single OmniRequestOutput + request = requests[0] + request_id = request.request_id or "" + prompt = request.prompt + if isinstance(prompt, list): + prompt = prompt[0] if prompt else None + + metrics = {} + if output.trajectory_timesteps is not None: + metrics["trajectory_timesteps"] = output.trajectory_timesteps + + if supports_audio_output(self.od_config.model_class_name): + audio_payload = outputs[0] if len(outputs) == 1 else outputs + return OmniRequestOutput.from_diffusion( + request_id=request_id, + images=[], + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + multimodal_output={"audio": audio_payload}, + final_output_type="audio", + ) + else: + return OmniRequestOutput.from_diffusion( + request_id=request_id, + images=outputs, + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + ) + else: + # Multiple requests: return list of OmniRequestOutput + # Split images based on num_outputs_per_prompt for each request + results = [] + output_idx = 0 + + for request in requests: + request_id = request.request_id or "" + prompt = request.prompt + if isinstance(prompt, list): + prompt = prompt[0] if prompt else None + + # Get images for this request + num_outputs = request.num_outputs_per_prompt + request_outputs = outputs[output_idx : output_idx + num_outputs] if output_idx < len(outputs) else [] + output_idx += num_outputs metrics = {} if output.trajectory_timesteps is not None: metrics["trajectory_timesteps"] = output.trajectory_timesteps if supports_audio_output(self.od_config.model_class_name): - audio_payload = outputs[0] if len(outputs) == 1 else outputs - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=[], - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, - multimodal_output={"audio": audio_payload}, - final_output_type="audio", + audio_payload = request_outputs[0] if len(request_outputs) == 1 else request_outputs + results.append( + OmniRequestOutput.from_diffusion( + request_id=request_id, + images=[], + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + multimodal_output={"audio": audio_payload}, + final_output_type="audio", + ) ) else: - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=outputs, - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, - ) - else: - # Multiple requests: return list of OmniRequestOutput - # Split images based on num_outputs_per_prompt for each request - results = [] - output_idx = 0 - - for request in requests: - request_id = request.request_id or "" - prompt = request.prompt - if isinstance(prompt, list): - prompt = prompt[0] if prompt else None - - # Get images for this request - num_outputs = request.num_outputs_per_prompt - request_outputs = ( - outputs[output_idx : output_idx + num_outputs] if output_idx < len(outputs) else [] - ) - output_idx += num_outputs - - metrics = {} - if output.trajectory_timesteps is not None: - metrics["trajectory_timesteps"] = output.trajectory_timesteps - - if supports_audio_output(self.od_config.model_class_name): - audio_payload = request_outputs[0] if len(request_outputs) == 1 else request_outputs - results.append( - OmniRequestOutput.from_diffusion( - request_id=request_id, - images=[], - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, - multimodal_output={"audio": audio_payload}, - final_output_type="audio", - ) - ) - else: - results.append( - OmniRequestOutput.from_diffusion( - request_id=request_id, - images=request_outputs, - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, - ) + results.append( + OmniRequestOutput.from_diffusion( + request_id=request_id, + images=request_outputs, + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, ) + ) - return results - except Exception as e: - logger.error(f"Generation failed: {e}") - return None + return results @staticmethod def make_engine(config: OmniDiffusionConfig) -> "DiffusionEngine": diff --git a/vllm_omni/diffusion/scheduler.py b/vllm_omni/diffusion/scheduler.py index 386218a61..21b0d4914 100644 --- a/vllm_omni/diffusion/scheduler.py +++ b/vllm_omni/diffusion/scheduler.py @@ -62,6 +62,9 @@ class Scheduler: raise RuntimeError("Result queue not initialized") output = self.result_mq.dequeue() + # {"status": "error", "error": str(e)} + if isinstance(output, dict) and output.get("status") == "error": + raise RuntimeError("worker error") return output except zmq.error.Again: logger.error("Timeout waiting for response from scheduler.") diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py b/vllm_omni/diffusion/worker/gpu_diffusion_worker.py index 5a4d4d016..90d169a0e 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py +++ b/vllm_omni/diffusion/worker/gpu_diffusion_worker.py @@ -279,7 +279,7 @@ class WorkerProc: return result, should_reply except Exception as e: logger.error(f"Error executing RPC: {e}", exc_info=True) - return {"status": "error", "error": str(e)}, should_reply + raise e def worker_busy_loop(self) -> None: """Main busy loop for Multiprocessing Workers.""" From 52bf1e2f400b32962bdaa31bcc40e1ad47b41d05 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Mon, 26 Jan 2026 11:21:24 +0800 Subject: [PATCH 15/83] [Feature]Support async computation and communication across stages by chunks (#727) Signed-off-by: amy-why-3459 Signed-off-by: Rein Yang Co-authored-by: Rein Yang Co-authored-by: CHEN <116010019@link.cuhk.edu.cn> Co-authored-by: Hongsheng Liu --- docs/api/README.md | 2 + .../examples/online_serving/qwen3_omni.md | 6 + examples/online_serving/qwen3_omni/README.md | 6 + ...letion_client_for_multimodal_generation.py | 97 +++-- vllm_omni/config/model.py | 12 + vllm_omni/core/sched/omni_ar_scheduler.py | 29 ++ .../core/sched/omni_generation_scheduler.py | 88 +++- vllm_omni/core/sched/output.py | 13 +- .../worker/gpu_diffusion_model_runner.py | 3 +- .../distributed/omni_connectors/adapter.py | 231 ++++++++++- .../omni_connectors/connectors/base.py | 13 +- .../connectors/shm_connector.py | 120 ++---- .../omni_connectors/utils/initialization.py | 2 + vllm_omni/engine/arg_utils.py | 32 +- vllm_omni/engine/output_processor.py | 8 +- vllm_omni/entrypoints/async_omni.py | 380 ++++++++++++------ vllm_omni/entrypoints/omni.py | 7 + vllm_omni/entrypoints/omni_stage.py | 23 +- vllm_omni/entrypoints/openai/serving_chat.py | 7 +- vllm_omni/entrypoints/stage_utils.py | 3 +- vllm_omni/entrypoints/utils.py | 2 + .../models/qwen3_omni/qwen3_omni.py | 176 +++++--- .../models/qwen3_omni/qwen3_omni_code2wav.py | 33 ++ .../stage_configs/qwen3_omni_moe.yaml | 8 +- .../qwen3_omni_moe_async_chunk.yaml | 101 +++++ .../stage_input_processors/qwen3_omni.py | 174 ++++++-- vllm_omni/worker/gpu_ar_model_runner.py | 8 +- .../worker/gpu_generation_model_runner.py | 13 + vllm_omni/worker/gpu_model_runner.py | 57 ++- 29 files changed, 1258 insertions(+), 396 deletions(-) create mode 100644 vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml diff --git a/docs/api/README.md b/docs/api/README.md index 258c3e7b5..3a110f491 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -53,8 +53,10 @@ Engine classes for offline and online inference. Core scheduling and caching components. +- [vllm_omni.core.sched.omni_ar_scheduler.KVCacheTransferData][] - [vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler][] - [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][] +- [vllm_omni.core.sched.output.OmniCachedRequestData][] - [vllm_omni.core.sched.output.OmniNewRequestData][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedGroupResidualVectorQuantization][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedResidualVectorQuantization][] diff --git a/docs/user_guide/examples/online_serving/qwen3_omni.md b/docs/user_guide/examples/online_serving/qwen3_omni.md index cd9198228..d26246533 100644 --- a/docs/user_guide/examples/online_serving/qwen3_omni.md +++ b/docs/user_guide/examples/online_serving/qwen3_omni.md @@ -15,6 +15,12 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` +If you want to open async chunking for qwen3-omni, launch the server with command below + +```bash +vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml +``` + If you have custom stage configs file, launch the server with command below ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file diff --git a/examples/online_serving/qwen3_omni/README.md b/examples/online_serving/qwen3_omni/README.md index 9c1f411f8..9db1060b7 100644 --- a/examples/online_serving/qwen3_omni/README.md +++ b/examples/online_serving/qwen3_omni/README.md @@ -12,6 +12,12 @@ Please refer to [README.md](../../../README.md) vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 ``` +If you want to open async chunking for qwen3-omni, launch the server with command below + +```bash +vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml +``` + If you have custom stage configs file, launch the server with command below ```bash vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file diff --git a/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py b/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py index e4768a8aa..304b27c4e 100644 --- a/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py +++ b/examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py @@ -1,4 +1,5 @@ import base64 +import concurrent.futures import os from typing import NamedTuple @@ -414,51 +415,67 @@ def run_multimodal_generation(args) -> None: else: output_modalities = None - chat_completion = client.chat.completions.create( - messages=[ - get_system_prompt(), - prompt, - ], - model=model_name, - modalities=output_modalities, - extra_body=extra_body, - stream=args.stream, - ) + # Test multiple concurrent completions + num_concurrent_requests = args.num_concurrent_requests + with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor: + # Submit multiple completion requests concurrently + futures = [ + executor.submit( + client.chat.completions.create, + messages=[ + get_system_prompt(), + prompt, + ], + model=model_name, + modalities=output_modalities, + extra_body=extra_body, + stream=args.stream, + ) + for _ in range(num_concurrent_requests) + ] + + # Wait for all requests to complete and collect results + chat_completions = [future.result() for future in concurrent.futures.as_completed(futures)] + + assert len(chat_completions) == num_concurrent_requests count = 0 if not args.stream: - for choice in chat_completion.choices: - if choice.message.audio: - audio_data = base64.b64decode(choice.message.audio.data) - audio_file_path = f"audio_{count}.wav" - with open(audio_file_path, "wb") as f: - f.write(audio_data) - print(f"Audio saved to {audio_file_path}") - count += 1 - elif choice.message.content: - print("Chat completion output from text:", choice.message.content) - else: - printed_content = False - for chunk in chat_completion: - for choice in chunk.choices: - if hasattr(choice, "delta"): - content = getattr(choice.delta, "content", None) - else: - content = None - - if getattr(chunk, "modality", None) == "audio" and content: - audio_data = base64.b64decode(content) + # Verify all completions succeeded + for chat_completion in chat_completions: + for choice in chat_completion.choices: + if choice.message.audio: + audio_data = base64.b64decode(choice.message.audio.data) audio_file_path = f"audio_{count}.wav" with open(audio_file_path, "wb") as f: f.write(audio_data) - print(f"\nAudio saved to {audio_file_path}") + print(f"Audio saved to {audio_file_path}") count += 1 + elif choice.message.content: + print("Chat completion output from text:", choice.message.content) + else: + printed_content = False + for chat_completion in chat_completions: + for chunk in chat_completion: + for choice in chunk.choices: + if hasattr(choice, "delta"): + content = getattr(choice.delta, "content", None) + else: + content = None - elif getattr(chunk, "modality", None) == "text": - if not printed_content: - printed_content = True - print("\ncontent:", end="", flush=True) - print(content, end="", flush=True) + if getattr(chunk, "modality", None) == "audio" and content: + audio_data = base64.b64decode(content) + audio_file_path = f"audio_{count}.wav" + with open(audio_file_path, "wb") as f: + f.write(audio_data) + print(f"\nAudio saved to {audio_file_path}") + count += 1 + + elif getattr(chunk, "modality", None) == "text": + if not printed_content: + printed_content = True + print("\ncontent:", end="", flush=True) + print(content, end="", flush=True) def parse_args(): @@ -517,6 +534,12 @@ def parse_args(): action="store_true", help="Stream the response.", ) + parser.add_argument( + "--num-concurrent-requests", + type=int, + default=1, + help="Number of concurrent requests to send. Default is 1.", + ) return parser.parse_args() diff --git a/vllm_omni/config/model.py b/vllm_omni/config/model.py index da0f5716b..4fe136288 100644 --- a/vllm_omni/config/model.py +++ b/vllm_omni/config/model.py @@ -1,4 +1,5 @@ import warnings +from dataclasses import field from typing import Any import torch @@ -42,6 +43,7 @@ class OmniModelConfig(ModelConfig): Attributes: stage_id: Identifier for the stage in a multi-stage pipeline (default: 0) + async_chunk: If set to True, perform async chunk model_stage: Stage type identifier, e.g., "thinker" or "talker" (default: "thinker") model_arch: Model architecture name @@ -49,6 +51,8 @@ class OmniModelConfig(ModelConfig): engine_output_type: Optional output type specification for the engine. Used to route outputs to appropriate processors (e.g., "image", "audio", "latents"). If None, output type is inferred. + stage_connector_config: Stage connector configuration dictionary. + Contains "name" (connector name), "extra" (extra connector config). Example: >>> config = OmniModelConfig( @@ -59,10 +63,18 @@ class OmniModelConfig(ModelConfig): """ stage_id: int = 0 + async_chunk: bool = False model_stage: str = "thinker" model_arch: str = "Qwen2_5OmniForConditionalGeneration" engine_output_type: str | None = None hf_config_name: str | None = None + custom_process_next_stage_input_func: str | None = None + stage_connector_config: dict[str, Any] = field( + default_factory=lambda: { + "name": "SharedMemoryConnector", + "extra": {}, + } + ) omni_kv_config: dict | None = None @property diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 46717cf04..03eb5ff21 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib from collections import defaultdict from dataclasses import asdict, dataclass from time import time @@ -19,6 +20,10 @@ from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import Request, RequestStatus from vllm.v1.spec_decode.metrics import SpecDecodingStats +from vllm_omni.distributed.omni_connectors.adapter import get_chunk, put_chunk +from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory +from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec + logger = init_logger(__name__) @@ -59,6 +64,25 @@ class OmniARScheduler(VLLMScheduler): # Track requests that have already triggered prefill transfer to avoid duplicates self.transfer_triggered_requests: set[str] = set() + model_config = self.vllm_config.model_config + self.omni_connector = None + if model_config.async_chunk: + connector_config = model_config.stage_connector_config + connector_specs = ConnectorSpec( + name=connector_config.get("name", "SharedMemoryConnector"), + extra=connector_config.get("extra", {}), + ) + self.omni_connector = OmniConnectorFactory.create_connector(connector_specs) + + custom_process_next_stage_input_func = getattr( + self.vllm_config.model_config, "custom_process_next_stage_input_func", None + ) + if custom_process_next_stage_input_func: + module_path, func_name = custom_process_next_stage_input_func.rsplit(".", 1) + module = importlib.import_module(module_path) + self.custom_process_next_stage_input_func = getattr(module, func_name) + + self.stage_id = getattr(self.vllm_config.model_config, "stage_id", None) def _get_kv_transfer_criteria(self) -> dict | None: # Note: vllm_config is available in Scheduler after super().__init__ @@ -156,6 +180,8 @@ class OmniARScheduler(VLLMScheduler): new_list.append(omni_nr) scheduler_output.scheduled_new_reqs = new_list # type: ignore[assignment] + if self.omni_connector is not None: + get_chunk(self.omni_connector, scheduler_output) # Add information about requests needing KV cache transfer scheduler_output.finished_requests_needing_kv_transfer = self.get_finished_requests_needing_kv_transfer() @@ -337,6 +363,9 @@ class OmniARScheduler(VLLMScheduler): num_nans_in_logits=request.num_nans_in_logits, ) ) + if self.omni_connector is not None: + custom_process_next_stage_input_func = self.custom_process_next_stage_input_func + put_chunk(self.omni_connector, pooler_output, request, custom_process_next_stage_input_func) else: # Invariant: EngineCore returns no partial prefill outputs. assert not prompt_logprobs_tensors diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index d9a4b12d3..0164029d2 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -12,11 +12,27 @@ from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutp from vllm.v1.request import Request, RequestStatus from vllm.v1.spec_decode.metrics import SpecDecodingStats -from vllm_omni.core.sched.output import OmniNewRequestData +from vllm_omni.core.sched.output import OmniCachedRequestData, OmniNewRequestData +from vllm_omni.distributed.omni_connectors.adapter import get_chunk_for_generation +from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory +from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec from vllm_omni.outputs import OmniModelRunnerOutput class OmniGenerationScheduler(VLLMScheduler): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + model_config = self.vllm_config.model_config + self.omni_connector = None + if model_config.async_chunk: + connector_config = model_config.stage_connector_config + connector_specs = ConnectorSpec( + name=connector_config.get("name", "SharedMemoryConnector"), + extra=connector_config.get("extra", {}), + ) + self.omni_connector = OmniConnectorFactory.create_connector(connector_specs) + self.stage_id = getattr(self.vllm_config.model_config, "stage_id", None) + def schedule(self) -> SchedulerOutput: """Diffusion fast path: - Feed all input tokens of the request at once @@ -32,27 +48,54 @@ class OmniGenerationScheduler(VLLMScheduler): req_to_new_blocks: dict[str, KVCacheBlocks] = {} num_scheduled_tokens: dict[str, int] = {} + scheduled_running_reqs: list[Request] = [] scheduled_spec_decode_tokens: dict[str, list[int]] = {} scheduled_encoder_inputs: dict[str, list[int]] = {} + cached_prompt_token_ids: dict[str, list[int]] = {} # Temporary queue: preserve waiting order, do not disturb non-diffusion requests skipped_waiting_requests = create_request_queue(self.policy) + req_index = 0 + while req_index < len(self.running) and token_budget > 0: + request = self.running[req_index] + if self.omni_connector is not None: + get_chunk_for_generation(self.omni_connector, request) + num_computed_tokens = request.num_computed_tokens + required_tokens = max(len(request.prompt_token_ids) - num_computed_tokens, 1) + num_new_tokens = min(required_tokens, token_budget) + new_blocks = self.kv_cache_manager.allocate_slots( + request, + num_new_tokens, + num_lookahead_tokens=self.num_lookahead_tokens, + ) + if new_blocks is None: + # Allocation failed (e.g., VRAM pressure); stop fast path and + # fall back to default scheduling + # Put the current request back to the head of the waiting queue + # Note: the original queue order is preserved + break + if self.log_stats: + request.record_event(EngineCoreEventType.SCHEDULED, scheduled_timestamp) + req_to_new_blocks[request.request_id] = new_blocks + num_scheduled_tokens[request.request_id] = num_new_tokens + cached_prompt_token_ids[request.request_id] = request.prompt_token_ids + token_budget -= num_new_tokens + scheduled_running_reqs.append(request) + req_index += 1 # Fast path selection and scheduling (treat all as diffusion requests, # independent of pooling_params) while self.waiting and token_budget > 0 and len(self.running) < self.max_num_running_reqs: request = self.waiting.peek_request() + if self.omni_connector is not None: + get_chunk_for_generation(self.omni_connector, request) # Uniformly treat as diffusion. A feature flag can be added later # via config or request tag. # Allocate all input tokens for the request in one shot # (allocate 1 placeholder if zero) - required_tokens = max(getattr(request, "num_prompt_tokens", 0), 1) - if required_tokens > token_budget: - # Insufficient budget to process all inputs at once; - # stop fast path attempt - break - num_new_tokens = required_tokens + required_tokens = max(len(request.prompt_token_ids), 1) + num_new_tokens = min(required_tokens, token_budget) new_blocks = self.kv_cache_manager.allocate_slots( request, num_new_tokens, @@ -68,7 +111,6 @@ class OmniGenerationScheduler(VLLMScheduler): # Officially schedule this request request = self.waiting.pop_request() self.running.append(request) - request.status = RequestStatus.RUNNING if self.log_stats: request.record_event(EngineCoreEventType.SCHEDULED, scheduled_timestamp) @@ -109,13 +151,24 @@ class OmniGenerationScheduler(VLLMScheduler): ] # No running/resumed reqs scheduled in our fast path cached_reqs_data = self._make_cached_request_data( - running_reqs=[], + running_reqs=scheduled_running_reqs, resumed_reqs=[], num_scheduled_tokens=num_scheduled_tokens, spec_decode_tokens=scheduled_spec_decode_tokens, req_to_new_blocks=req_to_new_blocks, ) + cached_reqs_data = OmniCachedRequestData( + req_ids=cached_reqs_data.req_ids, + resumed_req_ids=cached_reqs_data.resumed_req_ids, + new_token_ids=cached_reqs_data.new_token_ids, + all_token_ids=cached_reqs_data.all_token_ids, + new_block_ids=cached_reqs_data.new_block_ids, + num_computed_tokens=cached_reqs_data.num_computed_tokens, + num_output_tokens=cached_reqs_data.num_output_tokens, + prompt_token_ids=cached_prompt_token_ids, + ) + total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) scheduler_output = SchedulerOutput( scheduled_new_reqs=new_reqs_data, @@ -257,21 +310,20 @@ class OmniGenerationScheduler(VLLMScheduler): new_logprobs = None new_token_ids = generated_token_ids kv_transfer_params = None - status_before_stop = request.status pooler_output = None if pooler_outputs: pooler_output = pooler_outputs[req_index] # Diffusion request: completes in one step; mark finished and free resources - request.status = RequestStatus.FINISHED_STOPPED - # Optional: set a stop_reason for front-end clarity - # (does not affect protocol) - request.stop_reason = request.stop_reason # or "generation_done" - kv_transfer_params = self._free_request(request) - if status_before_stop == RequestStatus.RUNNING: + if request.status == RequestStatus.FINISHED_STOPPED or ( + self.omni_connector is None and request.num_computed_tokens >= request.num_prompt_tokens + ): + request.status = RequestStatus.FINISHED_STOPPED + # Optional: set a stop_reason for front-end clarity + # (does not affect protocol) + request.stop_reason = request.stop_reason # or "generation_done" + kv_transfer_params = self._free_request(request) stopped_running_reqs.add(request) - else: - stopped_preempted_reqs.add(request) # Extract sample logprobs if needed. if request.sampling_params is not None and request.sampling_params.logprobs is not None and logprobs: diff --git a/vllm_omni/core/sched/output.py b/vllm_omni/core/sched/output.py index 205513ac9..3176ec14a 100644 --- a/vllm_omni/core/sched/output.py +++ b/vllm_omni/core/sched/output.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from vllm.v1.core.sched.output import NewRequestData +from vllm.v1.core.sched.output import CachedRequestData, NewRequestData from vllm.v1.request import Request from vllm_omni.engine import AdditionalInformationPayload, PromptEmbedsPayload @@ -53,3 +53,14 @@ class OmniNewRequestData(NewRequestData): prefill_token_ids=prefill_token_ids, additional_information=request.additional_information, ) + + +@dataclass +class OmniCachedRequestData(CachedRequestData): + """Cached request data for omni models with embeddings support. + + Args: + prompt_token_ids: Mapping from request ID to list of prompt token IDs + """ + + prompt_token_ids: dict[int, list[int]] diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py b/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py index 5fb0cc670..54327be81 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py @@ -207,10 +207,11 @@ class GPUDiffusionModelRunner: start_time = time.time() while True: + get_key = f"omni_{from_stage}_to_{to_stage}_kv_cache_{req.request_id}" result = self.connector.get( from_stage=from_stage, to_stage=to_stage, - request_id=f"kv_cache_{req.request_id}", + get_key=get_key, ) if result: break diff --git a/vllm_omni/distributed/omni_connectors/adapter.py b/vllm_omni/distributed/omni_connectors/adapter.py index cfc959017..da3abf4a9 100644 --- a/vllm_omni/distributed/omni_connectors/adapter.py +++ b/vllm_omni/distributed/omni_connectors/adapter.py @@ -5,7 +5,14 @@ import time from collections.abc import Callable -from typing import Any +from typing import TYPE_CHECKING, Any + +import torch +from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.request import Request, RequestStatus + +if TYPE_CHECKING: + from .connectors.base import OmniConnectorBase from vllm_omni.entrypoints.stage_utils import OmniStageTaskType @@ -170,3 +177,225 @@ def try_recv_via_connector( # but for Stage-0 seed it should be there. # We'll return None to let caller handle error if strictly required. return None, None + + +def get_chunk( + connector: "OmniConnectorBase", + scheduler_output: SchedulerOutput, +) -> None: + """Retrieve a chunk of pooling output. + + Args: + connector: OmniConnectorBase instance + scheduler_output: Partial scheduler output dictionary + + Returns: + None: This function modifies scheduler_output in place + """ + stage_id = connector.stage_id + if stage_id == 0: + return + + target_stage_id = stage_id - 1 + # Handle new requests + for new_req_data in scheduler_output.scheduled_new_reqs: + req_id = new_req_data.req_id[0:25] + chunk_id = connector.get_requests[req_id] + connector_get_key = f"{req_id}_{target_stage_id}_{chunk_id}" + payload_data = get_through_connector(connector, target_stage_id, stage_id, req_id, connector_get_key) + if payload_data: + new_req_data.additional_information = payload_data + if payload_data.get("finished"): + connector.finished_requests.add(req_id) + + # Handle cached/running requests + cached_reqs = scheduler_output.scheduled_cached_reqs + if not hasattr(cached_reqs, "additional_information"): + cached_reqs.additional_information = {} + + for i, cached_req_id in enumerate(cached_reqs.req_ids): + req_id = cached_req_id[0:25] + if req_id in connector.finished_requests: + continue + chunk_id = connector.get_requests[req_id] + connector_get_key = f"{req_id}_{target_stage_id}_{chunk_id}" + payload_data = get_through_connector(connector, target_stage_id, stage_id, req_id, connector_get_key) + if payload_data: + cached_reqs.additional_information[cached_req_id] = payload_data + if payload_data.get("finished"): + connector.finished_requests.add(req_id) + + +def get_through_connector(connector, target_stage_id, stage_id, req_id, connector_get_key): + # Wait for data from previous stage + import time + + # TODO: add correct check mechanism for the payload_data + max_wait = 300 + for _ in range(max_wait): + result = connector.get( + from_stage=str(target_stage_id), + to_stage=str(stage_id), + get_key=connector_get_key, + ) + payload_data = None + if result: + payload_data, size = result + if payload_data: + connector.request_prompt_token_ids[req_id] = payload_data.get("thinker_input_ids", []) + connector.get_requests[req_id] += 1 + logger.debug(f"[Stage-{stage_id}] Received one chunk for request {connector_get_key}") + break + time.sleep(0.1) + return payload_data + + +def get_chunk_for_generation( + connector: "OmniConnectorBase", + request: Request, +) -> None: + """Retrieve a chunk of pooling output. + + Args: + connector: OmniConnectorBase instance + request: Request object + + Returns: + None: This function modifies request in place + """ + stage_id = connector.stage_id + target_stage_id = stage_id - 1 + request_id = request.request_id[0:25] + + if request_id in connector.finished_requests: + return + + chunk_id = connector.get_requests[request_id] + connector_get_key = f"{request_id}_{target_stage_id}_{chunk_id}" + payload_data = get_through_connector(connector, target_stage_id, stage_id, request_id, connector_get_key) + if not payload_data: + return + + if payload_data.get("finished"): + connector.finished_requests.add(request_id) + request.status = RequestStatus.FINISHED_STOPPED + + # TODO: remove special handling for prompt token ids ? + if chunk_id == 0: + request.prompt_token_ids = payload_data.get("code_predictor_codes", []) + else: + request.prompt_token_ids += payload_data.get("code_predictor_codes", []) + + +def put_chunk( + connector: "OmniConnectorBase", + pooling_output: dict[str, Any], + request: Request, + custom_process_input_func: Callable[[dict[str, Any], Request], dict[str, Any] | None] | None = None, +) -> None: + """Store a chunk of pooling output. + + Args: + connector: OmniConnectorBase instance + pooling_output: Partial pooling output dictionary + request: Request object + custom_process_input_func: Optional custom function to process input + + Returns: + None: This function sends data via connector + """ + stage_id = connector.stage_id + next_stage_id = stage_id + 1 + request_id = request.request_id[0:25] + prompt_token_ids = request.prompt_token_ids + connector.request_prompt_token_ids[request_id] = prompt_token_ids + chunk_id = connector.put_requests[request_id] + connector_put_key = f"{request_id}_{stage_id}_{chunk_id}" + payload_data = None + + # TODO: add default process_input_func to handle the payload_data ? + if custom_process_input_func: + try: + payload_data = custom_process_input_func( + pooling_output=pooling_output, + request=request, + ) + except Exception as e: + logger.error(f"Failed to use custom_process_input_func for payload extraction: {e}") + + if not payload_data: + logger.warning(f"[Stage-{stage_id}] No payload data to send for request {request_id}") + return + + if stage_id == 0 and chunk_id == 0: + if connector.request_payload.get(request_id) is None: + connector.request_payload[request_id] = payload_data + return + else: + save_payload = connector.request_payload.get(request_id) + payload_data["thinker_embeddings"] = torch.cat( + (save_payload.get("thinker_embeddings"), payload_data.get("thinker_embeddings")), dim=0 + ) + payload_data["thinker_hidden_states"] = torch.cat( + (save_payload.get("thinker_hidden_states"), payload_data.get("thinker_hidden_states")), dim=0 + ) + logger.info(f"[Stage-{stage_id}] Merged embeddings and hidden states for request {request_id}") + + if stage_id == 1: + # TODO: Make parameters configurable and optimize algorithms + chunk_size = left_context_size = 25 + connector.code_prompt_token_ids[request_id].append(payload_data.get("code_predictor_codes", [])) + length = len(connector.code_prompt_token_ids[request_id]) + chunk_length = length % chunk_size + if chunk_length != 0 and not payload_data.get("finished"): + return + + context_length = chunk_length if chunk_length != 0 else chunk_size + end_index = min(length, left_context_size + context_length) + payload_data["code_predictor_codes"] = ( + torch.tensor(connector.code_prompt_token_ids[request_id][-end_index:]) + .transpose(0, 1) + .reshape(-1) + .tolist() + ) + + success, size, metadata = connector.put( + from_stage=str(stage_id), to_stage=str(next_stage_id), put_key=connector_put_key, data=payload_data + ) + + if success: + connector.put_requests[request_id] += 1 + logger.info(f"[Stage-{stage_id}] Sent {connector_put_key}") + + +def compute_talker_prompt_ids_length(prompt_ids: list[int]) -> int: + """Compute the length of the talker prompt ids. + + Args: + prompt_ids: The prompt ids tensor. + + Returns: + The length of the talker prompt ids. + """ + im_start_token_id = 151644 + system_token_id = 8948 + user_token_id = 872 + assistant_token_id = 77091 + im_start_indexes = [i for i in range(len(prompt_ids)) if prompt_ids[i] == im_start_token_id] + im_start_indexes.append(len(prompt_ids)) + sum_user_len = 0 + assistant_len = 0 + for i in range(len(im_start_indexes) - 1): + s = im_start_indexes[i] + e = im_start_indexes[i + 1] + role = prompt_ids[s + 1] + if role == system_token_id: + continue + elif role == user_token_id: + sum_user_len += e - s + elif role == assistant_token_id and i == len(im_start_indexes) - 2: + assistant_len += 9 # 3 + 4 + 1 + 1 + else: + pass + + return sum_user_len + assistant_len diff --git a/vllm_omni/distributed/omni_connectors/connectors/base.py b/vllm_omni/distributed/omni_connectors/connectors/base.py index 6163e7b86..e3562b896 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/base.py +++ b/vllm_omni/distributed/omni_connectors/connectors/base.py @@ -13,15 +13,13 @@ class OmniConnectorBase(ABC): """Base class for all OmniConnectors.""" @abstractmethod - def put( - self, from_stage: str, to_stage: str, request_id: str, data: Any - ) -> tuple[bool, int, dict[str, Any] | None]: + def put(self, from_stage: str, to_stage: str, put_key: str, data: Any) -> tuple[bool, int, dict[str, Any] | None]: """Store Python object, internal serialization handled by connector. Args: from_stage: Source stage identifier to_stage: Destination stage identifier - request_id: Unique request identifier + put_key: Unique request identifier data: Python object to store Returns: @@ -31,16 +29,13 @@ class OmniConnectorBase(ABC): pass @abstractmethod - def get( - self, from_stage: str, to_stage: str, request_id: str, metadata: dict[str, Any] | None = None - ) -> tuple[Any, int] | None: + def get(self, from_stage: str, to_stage: str, get_key: str, metadata=None) -> tuple[Any, int] | None: """Retrieve Python object and payload size (bytes). Args: from_stage: Source stage identifier to_stage: Destination stage identifier - request_id: Unique request identifier - metadata: Optional transport-specific metadata from the put operation + get_key: Unique request identifier Returns: Tuple of (Python object, serialized byte size) if found, None otherwise diff --git a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py index f65e29787..a9587e21c 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time +from collections import defaultdict from typing import Any from vllm_omni.entrypoints.stage_utils import shm_read_bytes, shm_write_bytes @@ -19,6 +21,14 @@ class SharedMemoryConnector(OmniConnectorBase): def __init__(self, config: dict[str, Any]): self.config = config + self.stage_id = config.get("stage_id", -1) + self.device = config.get("device", "cuda:0") + self.put_requests: dict[str, int] = defaultdict(int) + self.get_requests: dict[str, int] = defaultdict(int) + self.finished_requests: set[str] = set() + self.request_payload = {} + self.request_prompt_token_ids: dict[str, list[int]] = defaultdict(list) + self.code_prompt_token_ids: dict[str, list[list[int]]] = defaultdict(list) # Default threshold matches legacy behavior (64KB) self.threshold = int(config.get("shm_threshold_bytes", 65536)) self._metrics = { @@ -29,9 +39,7 @@ class SharedMemoryConnector(OmniConnectorBase): "inline_writes": 0, } - def put( - self, from_stage: str, to_stage: str, request_id: str, data: Any - ) -> tuple[bool, int, dict[str, Any] | None]: + def put(self, from_stage: str, to_stage: str, put_key: str, data: Any) -> tuple[bool, int, dict[str, Any] | None]: try: # Always serialize first to check size (and for SHM writing) # Note: For extremely large objects in "inline" mode (e.g. Ray), @@ -40,18 +48,19 @@ class SharedMemoryConnector(OmniConnectorBase): payload = self.serialize_obj(data) size = len(payload) - if size > self.threshold: + metadata = {} + # if size > self.threshold: + if True: # TODO: correct put & get logic # Use Shared Memory - shm_name = f"omni_{from_stage}_to_{to_stage}_{request_id}" - meta = shm_write_bytes(payload, name=shm_name) + meta = shm_write_bytes(payload, name=put_key) # meta contains {'name': ..., 'size': ...} - metadata = {"shm": meta, "size": size} + metadata[put_key] = {"shm": meta, "size": size} self._metrics["shm_writes"] += 1 else: # Inline - pass bytes directly to avoid double serialization of the object # We already serialized it to check size, so we pass the bytes. # The Queue will pickle these bytes (fast), avoiding re-serializing the complex object. - metadata = {"inline_bytes": payload, "size": size} + metadata[put_key] = {"inline_bytes": payload, "size": size} self._metrics["inline_writes"] += 1 self._metrics["puts"] += 1 @@ -60,83 +69,40 @@ class SharedMemoryConnector(OmniConnectorBase): return True, size, metadata except Exception as e: - logger.error(f"SharedMemoryConnector put failed for req {request_id}: {e}") + logger.error(f"SharedMemoryConnector put failed for req {put_key}: {e}") return False, 0, None - def get( - self, from_stage: str, to_stage: str, request_id: str, metadata: dict[str, Any] | None = None - ) -> tuple[Any, int] | None: - # Helper to try reading SHM with a specific name - def try_read_shm(name: str): - # _shm is imported at module level as: import multiprocessing.shared_memory as _shm - # but to be safe inside this closure if not picked up: - from multiprocessing import shared_memory as shm_pkg + def get(self, from_stage: str, to_stage: str, get_key: str, metadata=None) -> tuple[Any, int] | None: + from multiprocessing import shared_memory as shm_pkg - shm = shm_pkg.SharedMemory(name=name) + # Wait for shared memory to be available (with retry logic) + max_retries = 30 + retry_delay = 0.1 # 100ms between retries + shm = None + + for attempt in range(max_retries): try: - # We assume the buffer size matches the data size - return shm_read_bytes({"name": name, "size": shm.size}) - finally: - shm.close() + shm = shm_pkg.SharedMemory(name=get_key) + break # Successfully opened, exit retry loop + except FileNotFoundError: + if attempt < max_retries - 1: + time.sleep(retry_delay) + else: + # Max retries reached, return None + logger.warning(f"Shared memory '{get_key}' not found after {max_retries} retries") + return None, 0 - if not metadata: - # Try to infer metadata from request_id for KV cache transfer scenarios - # where metadata is not passed out-of-band. - shm_name = f"omni_{from_stage}_to_{to_stage}_{request_id}" - - # We need the size to read from SHM using stage_utils.shm_read_bytes - try: - # Use local import to avoid potential scope issues if _shm isn't available - from multiprocessing import shared_memory as shm_pkg - - temp_shm = shm_pkg.SharedMemory(name=shm_name) - size = temp_shm.size - temp_shm.close() - - metadata = {"shm": {"name": shm_name, "size": size}, "is_fallback": True} - logger.debug(f"SharedMemoryConnector inferred metadata for {request_id}: {metadata}") - except Exception as e: - # If file not found or other error, standard path will likely fail too, but we let it try or return None - logger.debug(f"SharedMemoryConnector failed to infer metadata for {request_id} (fallback): {e}") - # We can't proceed without size if we rely on shm_read_bytes - return None + if shm is None: + return None, 0 try: - obj = None - size = 0 + data_bytes = shm_read_bytes({"name": get_key, "size": shm.size}) + obj = self.deserialize_obj(data_bytes) + return obj, shm.size + finally: + shm.close() - if "shm" in metadata: - meta = metadata["shm"] - # shm_read_bytes handles reading and unlinking - data_bytes = shm_read_bytes(meta) - obj = self.deserialize_obj(data_bytes) - size = metadata.get("size", len(data_bytes)) - elif "inline_bytes" in metadata: - # Deserialize bytes back to object - payload = metadata["inline_bytes"] - obj = self.deserialize_obj(payload) - size = metadata.get("size", len(payload)) - elif "inline" in metadata: - obj = metadata["inline"] - size = metadata.get("size", 0) - if size == 0: - # Fallback if size wasn't recorded - try: - size = len(self.serialize_obj(obj)) - except Exception: - pass - else: - logger.error( - f"Unknown metadata format in SharedMemoryConnector for req {request_id}: {list(metadata.keys())}" - ) - return None - - self._metrics["gets"] += 1 - return obj, size - - except Exception as e: - logger.error(f"SharedMemoryConnector get failed for req {request_id}: {e}") - return None + # TODO: update another read method def cleanup(self, request_id: str) -> None: # SHM segments are automatically unlinked during 'get' (shm_read_bytes). diff --git a/vllm_omni/distributed/omni_connectors/utils/initialization.py b/vllm_omni/distributed/omni_connectors/utils/initialization.py index 60fe62c13..91322e63e 100644 --- a/vllm_omni/distributed/omni_connectors/utils/initialization.py +++ b/vllm_omni/distributed/omni_connectors/utils/initialization.py @@ -91,6 +91,8 @@ def get_connectors_config_for_stage(transfer_config: OmniTransferConfig | None, # (Worker needs to create connectors to receive data) if to_stage == target_stage: stage_connectors_config[f"from_stage_{from_stage}"] = {"spec": {"name": spec.name, "extra": spec.extra}} + elif from_stage == target_stage and target_stage == "0": + stage_connectors_config[f"to_stage_{to_stage}"] = {"spec": {"name": spec.name, "extra": spec.extra}} return stage_connectors_config diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index 3f2525ca8..cbe2001d7 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import Qwen3OmniMoeTextConfig from vllm.engine.arg_utils import EngineArgs @@ -56,6 +56,11 @@ class OmniEngineArgs(EngineArgs): engine_output_type: Optional output type specification for the engine. Used to route outputs to appropriate processors (e.g., "image", "audio", "latents"). If None, output type is inferred. + custom_process_next_stage_input_func: Optional path to a custom function for processing + inputs from previous stages + If None, default processing is used. + stage_connector_spec: Extra configuration for stage connector + async_chunk: If set to True, perform async chunk """ stage_id: int = 0 @@ -63,6 +68,9 @@ class OmniEngineArgs(EngineArgs): model_arch: str = "Qwen2_5OmniForConditionalGeneration" engine_output_type: str | None = None hf_config_name: str | None = None + custom_process_next_stage_input_func: str | None = None + stage_connector_spec: dict[str, any] = field(default_factory=dict) + async_chunk: bool = False omni_kv_config: dict | None = None def draw_hf_text_config(self, config_dict: dict) -> Qwen3OmniMoeTextConfig: @@ -112,10 +120,20 @@ class OmniEngineArgs(EngineArgs): # Add the new omni-specific fields config_dict["stage_id"] = self.stage_id + config_dict["async_chunk"] = self.async_chunk config_dict["model_stage"] = self.model_stage config_dict["model_arch"] = self.model_arch config_dict["engine_output_type"] = self.engine_output_type + # Build stage_connector_config from stage_connector_spec + stage_connector_config = { + "name": self.stage_connector_spec.get("name", "SharedMemoryConnector"), + "extra": self.stage_connector_spec.get("extra", {}).copy(), + } + stage_connector_config["extra"]["stage_id"] = self.stage_id + config_dict["stage_connector_config"] = stage_connector_config + config_dict["hf_config_name"] = self.hf_config_name + config_dict["custom_process_next_stage_input_func"] = self.custom_process_next_stage_input_func config_dict["omni_kv_config"] = self.omni_kv_config if self.hf_config_name is not None: config_dict["hf_text_config"] = self.draw_hf_text_config(config_dict) @@ -140,6 +158,7 @@ class AsyncOmniEngineArgs(AsyncEngineArgs): engine_output_type: Optional output type specification for the engine. Used to route outputs to appropriate processors (e.g., "image", "audio", "latents"). If None, output type is inferred. + stage_connector_spec: Extra configuration for stage connector """ stage_id: int = 0 @@ -147,6 +166,9 @@ class AsyncOmniEngineArgs(AsyncEngineArgs): model_arch: str = "Qwen2_5OmniForConditionalGeneration" engine_output_type: str | None = None hf_config_name: str | None = None + custom_process_next_stage_input_func: str | None = None + stage_connector_spec: dict[str, any] = field(default_factory=dict) + async_chunk: bool = False omni_kv_config: dict | None = None def draw_hf_text_config(self, config_dict: dict) -> Qwen3OmniMoeTextConfig: @@ -186,11 +208,19 @@ class AsyncOmniEngineArgs(AsyncEngineArgs): # Add the new omni-specific fields config_dict["stage_id"] = self.stage_id + config_dict["async_chunk"] = self.async_chunk config_dict["model_stage"] = self.model_stage config_dict["model_arch"] = self.model_arch config_dict["engine_output_type"] = self.engine_output_type + stage_connector_config = { + "name": self.stage_connector_spec.get("name", "SharedMemoryConnector"), + "extra": self.stage_connector_spec.get("extra", {}).copy(), + } + stage_connector_config["extra"]["stage_id"] = self.stage_id + config_dict["stage_connector_config"] = stage_connector_config config_dict["hf_config_name"] = self.hf_config_name + config_dict["custom_process_next_stage_input_func"] = self.custom_process_next_stage_input_func config_dict["omni_kv_config"] = self.omni_kv_config if self.hf_config_name is not None: config_dict["hf_text_config"] = self.draw_hf_text_config(config_dict) diff --git a/vllm_omni/engine/output_processor.py b/vllm_omni/engine/output_processor.py index 6aef7fb16..ab5cf878c 100644 --- a/vllm_omni/engine/output_processor.py +++ b/vllm_omni/engine/output_processor.py @@ -111,9 +111,15 @@ class OmniRequestState(RequestState): for k, v in self.mm_accumulated.items(): if isinstance(v, list) and v and isinstance(v[0], torch.Tensor): try: - self.mm_accumulated[k] = torch.cat(v, dim=0) + if k == "audio": + # When the audio tensor shape is inconsistent, torch.cat will fail. + # We need to use torch.cat in -1 dimension. + self.mm_accumulated[k] = torch.cat(v, dim=-1) + else: + self.mm_accumulated[k] = torch.cat(v, dim=0) except Exception: # Keep last tensor on failure + logger.warning(f"Error concatenating tensor for key {k}; keeping last tensor") self.mm_accumulated[k] = v[-1] elif isinstance(v, dict): for sk, sv in v.items(): diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 3c275147f..88a290cdf 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -20,7 +20,7 @@ from vllm.v1.engine.exceptions import EngineDeadError # Internal imports (our code) from vllm_omni.config import OmniModelConfig from vllm_omni.diffusion.data import DiffusionParallelConfig -from vllm_omni.distributed.omni_connectors.adapter import try_send_via_connector +from vllm_omni.distributed.omni_connectors.adapter import compute_talker_prompt_ids_length, try_send_via_connector from vllm_omni.distributed.ray_utils.utils import try_close_ray from vllm_omni.engine.input_processor import OmniInputProcessor from vllm_omni.entrypoints.client_request_state import ClientRequestState @@ -325,139 +325,76 @@ class AsyncOmni(OmniBase): self._enable_stats, _wall_start_ts, ) - # Seed stage-0 queue with all requests - logger.debug(f"[{self._name}] Seeding request into stage-0") req_state = ClientRequestState(request_id) req_state.metrics = metrics self.request_states[request_id] = req_state - # Mark first input time for stage-0 - metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time() - sp0: SamplingParams = sampling_params_list[0] # type: ignore[index] - task = { - "request_id": request_id, - "engine_inputs": prompt, - "sampling_params": sp0, - } - self.stage_list[0].submit(task) - _req_start_ts[request_id] = time.time() - logger.debug(f"[{self._name}] Enqueued request {request_id} to stage-0") + if self.async_chunk: + stage_queues = {stage_id: asyncio.Queue() for stage_id in range(num_stages)} + req_state.stage_queues = stage_queues + for i in range(num_stages): + sp: SamplingParams = sampling_params_list[i] + engine_inputs = prompt + if i != 0: + prompt_token_ids = prompt["prompt_token_ids"] + prompt_1 = prompt.copy() + prompt_1["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids) + prompt_1["multi_modal_data"] = prompt_1["mm_processor_kwargs"] = None + engine_inputs = prompt_1 - logger.debug(f"[{self._name}] Entering scheduling loop: stages={num_stages}") - for stage_id, stage in enumerate(self.stage_list[: final_stage_id_for_e2e + 1]): - finished = False - while not finished: - result = await req_state.queue.get() - assert stage_id == req_state.stage_id + task = { + "request_id": request_id, + "engine_inputs": engine_inputs, + "sampling_params": sp, + } + self.stage_list[i].submit(task) + metrics.stage_first_ts[i] = metrics.stage_first_ts[0] or time.time() - req_id = result.get("request_id") - if "error" in result: - logger.error( - f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}", - ) - raise RuntimeError(result) # Request Finished due to error + logger.info(f"[{self._name}] Enqueued request {request_id} to stage-{str(i)}") - engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm") - if isinstance(engine_outputs, list): - engine_outputs = engine_outputs[0] - finished = engine_outputs.finished + _req_start_ts[request_id] = time.time() - # Mark last output time for this stage whenever we receive outputs - metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time()) - try: - _m = asdict(result.get("metrics")) - if _m is not None and finished: - metrics.on_stage_metrics(stage_id, req_id, _m) - except Exception as e: - logger.exception( - f"[{self._name}] Failed to process metrics for stage {stage_id}, req {req_id}: {e}", - ) - logger.debug( - f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing", - ) + logger.info( + f"[{self._name}] Entering scheduling loop: " + f"stages={num_stages}, final_stage={final_stage_id_for_e2e}" + ) + async for output in self._process_async_results( + request_id, + req_state, + metrics, + final_stage_id_for_e2e, + _req_start_ts, + _wall_start_ts, + ): + yield output + else: + sp0: SamplingParams = sampling_params_list[0] # type: ignore[index] + task = { + "request_id": request_id, + "engine_inputs": prompt, + "sampling_params": sp0, + } + self.stage_list[0].submit(task) - if getattr(stage, "final_output", False): - logger.debug( - f"[{self._name}] Request {req_id} finalized at stage-{stage_id}", - ) + _req_start_ts[request_id] = time.time() + # Mark first input time for stage-0 + metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time() + logger.info( + f"[{self._name}] Entering scheduling loop: " + f"stages={num_stages}, final_stage={final_stage_id_for_e2e}" + ) - # End-to-end timing and time-per-token for final output - # (only once per request at the designated final stage) - try: - rid_key = str(req_id) - if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done and finished: - metrics.on_finalize_request( - stage_id, - req_id, - _req_start_ts.get(req_id, _wall_start_ts), - ) - except Exception as e: - logger.exception( - f"[{self._name}] Finalize request handling error for req " - f"{req_id} at stage {stage_id}: {e}", - ) - - # Handle diffusion outputs that already contain images - if stage.final_output_type == "image": - images = [] - if isinstance(engine_outputs, OmniRequestOutput) and engine_outputs.images: - images = engine_outputs.images - elif hasattr(engine_outputs, "images") and engine_outputs.images: - images = engine_outputs.images - yield OmniRequestOutput( - stage_id=stage_id, - final_output_type=stage.final_output_type, - request_output=engine_outputs, - images=images, - ) - else: - yield OmniRequestOutput( - stage_id=stage_id, - final_output_type=stage.final_output_type, - request_output=engine_outputs, - ) - if not isinstance(engine_outputs, list): - engine_outputs = [engine_outputs] - stage.set_engine_outputs(engine_outputs) - # Forward to next stage if there is one - next_stage_id = stage_id + 1 - if next_stage_id <= final_stage_id_for_e2e and finished: - next_stage: OmniStage = self.stage_list[next_stage_id] - next_inputs = next_stage.process_engine_inputs(self.stage_list, prompt) - sp_next: SamplingParams = sampling_params_list[next_stage_id] - - # Check if we have a connector for this edge - connector_key = (str(stage_id), str(next_stage_id)) - connector = self.connectors.get(connector_key) - - sent_via_connector = False - if connector: - sent_via_connector = try_send_via_connector( - connector=connector, - stage_id=stage_id, - next_stage_id=next_stage_id, - req_id=req_id, - next_inputs=next_inputs, - sampling_params=sp_next, - original_prompt=prompt, - next_stage_queue_submit_fn=self.stage_list[next_stage_id].submit, - metrics=metrics, - ) - - if not sent_via_connector: - # Fallback logic removed as we now enforce connector usage. - # If no connector is found or send fails, we log an error and raise, - # because continuing would cause the request to be silently dropped - # and the orchestrator to hang waiting for completion. - error_msg = ( - f"[{self._name}] Failed to send request {req_id} to stage-{next_stage_id} via connector. " - "Configure a connector for this edge or inspect connector logs for details." - ) - logger.error(error_msg) - raise RuntimeError(error_msg) - logger.debug(f"[{self._name}] Forwarded request {req_id} to stage-{next_stage_id}") - else: - logger.debug(f"[{self._name}] Request {req_id} fully completed") + async for output in self._process_sequential_results( + request_id, + req_state, + metrics, + final_stage_id_for_e2e, + _req_start_ts, + _wall_start_ts, + sampling_params_list, + prompt, + ): + yield output logger.debug(f"[{self._name}] All requests completed") @@ -474,6 +411,184 @@ class AsyncOmni(OmniBase): logger.info("[AsyncOrchestrator] Request %s aborted.", request_id) raise + async def _process_async_results( + self, + request_id: str, + req_state: ClientRequestState, + metrics: OrchestratorMetrics, + final_stage_id_for_e2e: int, + req_start_ts: dict[int, float], + wall_start_ts: float, + ) -> AsyncGenerator[OmniRequestOutput, None]: + all_stages_finished = {stage_id: False for stage_id in range(final_stage_id_for_e2e + 1)} + while not all(all_stages_finished.values()): + for stage_id, stage in enumerate(self.stage_list[: final_stage_id_for_e2e + 1]): + if all_stages_finished[stage_id]: + continue + result = await req_state.stage_queues[stage_id].get() + logger.info(f"[{self._name}] Received result from stage-{stage_id}: {result}") + engine_outputs, finished, output_to_yield = self._process_single_result( + result, stage, stage_id, metrics, req_start_ts, wall_start_ts, final_stage_id_for_e2e + ) + + all_stages_finished[stage_id] = finished + + if output_to_yield: + yield output_to_yield + + async def _process_sequential_results( + self, + request_id: str, + req_state: ClientRequestState, + metrics: OrchestratorMetrics, + final_stage_id_for_e2e: int, + req_start_ts: dict[int, float], + wall_start_ts: float, + sampling_params_list: list[SamplingParams], + prompt: Any, + ) -> AsyncGenerator[OmniRequestOutput, None]: + for stage_id, stage in enumerate(self.stage_list[: final_stage_id_for_e2e + 1]): + finished = False + while not finished: + result = await req_state.queue.get() + assert stage_id == req_state.stage_id + req_id = result.get("request_id") + engine_outputs, finished, output_to_yield = self._process_single_result( + result, stage, stage_id, metrics, req_start_ts, wall_start_ts, final_stage_id_for_e2e + ) + if output_to_yield: + yield output_to_yield + if not isinstance(engine_outputs, list): + engine_outputs = [engine_outputs] + stage.set_engine_outputs(engine_outputs) + # Forward to next stage if there is one + next_stage_id = stage_id + 1 + if next_stage_id <= final_stage_id_for_e2e and finished: + next_stage: OmniStage = self.stage_list[next_stage_id] + next_inputs = next_stage.process_engine_inputs(self.stage_list, prompt) + sp_next: SamplingParams = sampling_params_list[next_stage_id] + + # Check if we have a connector for this edge + connector_key = (str(stage_id), str(next_stage_id)) + connector = self.connectors.get(connector_key) + + sent_via_connector = False + if connector: + sent_via_connector = try_send_via_connector( + connector=connector, + stage_id=stage_id, + next_stage_id=next_stage_id, + req_id=req_id, + next_inputs=next_inputs, + sampling_params=sp_next, + original_prompt=prompt, + next_stage_queue_submit_fn=self.stage_list[next_stage_id].submit, + metrics=metrics, + ) + + if not sent_via_connector: + # Fallback logic removed as we now enforce connector usage. + # If no connector is found or send fails, we log an error and raise, + # because continuing would cause the request to be silently dropped + # and the orchestrator to hang waiting for completion. + error_msg = ( + f"[{self._name}] Failed to send request {req_id} to stage-{next_stage_id} via connector. " + "Configure a connector for this edge or inspect connector logs for details." + ) + logger.error(error_msg) + raise RuntimeError(error_msg) + logger.debug(f"[{self._name}] Forwarded request {req_id} to stage-{next_stage_id}") + else: + logger.debug(f"[{self._name}] Request {req_id} fully completed") + + def _process_single_result( + self, + result: dict[str, Any], + stage: OmniStage, + stage_id: int, + metrics: OrchestratorMetrics, + req_start_ts: dict[int, float], + wall_start_ts: float, + final_stage_id_for_e2e: int, + ) -> tuple[Any, bool, OmniRequestOutput | None]: + """ + Process a single result dictionary from a stage. + Returns: + engine_outputs: The decoded outputs. + finished: Whether the stage processing is finished for this request. + output_to_yield: An OmniRequestOutput to yield, or None. + """ + req_id = result.get("request_id") + if "error" in result: + logger.error( + f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}", + ) + raise RuntimeError(result) + + engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm") + if isinstance(engine_outputs, list): + engine_outputs = engine_outputs[0] + + finished = engine_outputs.finished + + # Mark last output time + metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time()) + + try: + _m = asdict(result.get("metrics")) + if _m is not None and finished: + metrics.on_stage_metrics(stage_id, req_id, _m) + except Exception as e: + logger.exception( + f"[{self._name}] Failed to process metrics for stage {stage_id}, req {req_id}: {e}", + ) + + logger.debug( + f"[{self._name}] Stage-{stage_id} completed request {req_id}; forwarding or finalizing", + ) + + output_to_yield = None + if getattr(stage, "final_output", False): + logger.debug(f"[{self._name}] Request {req_id} finalized at stage-{stage_id}") + + # Finalize request metrics if this is the E2E final stage and it's finished + try: + rid_key = str(req_id) + if stage_id == final_stage_id_for_e2e and rid_key not in metrics.e2e_done and finished: + metrics.on_finalize_request( + stage_id, + req_id, + req_start_ts.get(req_id, wall_start_ts), + ) + except Exception as e: + logger.exception( + f"[{self._name}] Finalize request handling error for req {req_id} at stage {stage_id}: {e}", + ) + + # Construct output to yield + images = [] + if stage.final_output_type == "image": + if isinstance(engine_outputs, OmniRequestOutput) and engine_outputs.images: + images = engine_outputs.images + elif hasattr(engine_outputs, "images") and engine_outputs.images: + images = engine_outputs.images + + if stage.final_output_type == "image": + output_to_yield = OmniRequestOutput( + stage_id=stage_id, + final_output_type=stage.final_output_type, + request_output=engine_outputs, + images=images, + ) + else: + output_to_yield = OmniRequestOutput( + stage_id=stage_id, + final_output_type=stage.final_output_type, + request_output=engine_outputs, + ) + + return engine_outputs, finished, output_to_yield + def _run_output_handler(self) -> None: if self.output_handler is not None: return @@ -503,8 +618,12 @@ class AsyncOmni(OmniBase): dropping output for req {req_id} at stage-{stage_id}" ) continue - await req_state.queue.put(result) - req_state.stage_id = stage_id + if hasattr(req_state, "stage_queues") and stage_id in req_state.stage_queues: + await req_state.stage_queues[stage_id].put(result) + else: + # Fallback to old behavior for compatibility + await req_state.queue.put(result) + req_state.stage_id = stage_id if idle: await asyncio.sleep(0.001) # Avoid CPU overload when idle else: @@ -512,7 +631,14 @@ class AsyncOmni(OmniBase): except Exception as e: logger.exception("AsyncOmni output_handler failed.") for req_state in request_states.values(): - await req_state.queue.put({"request_id": req_id, "error": str(e)}) + error_msg = {"request_id": req_state.request_id, "error": str(e)} + # Send error to all stage queues + if hasattr(req_state, "stage_queues"): + for queue in req_state.stage_queues.values(): + await queue.put(error_msg) + else: + await req_state.queue.put(error_msg) + error_msg = {"request_id": req_state.request_id, "error": str(e)} self.output_handler = None # Make possible for restart self.output_handler = asyncio.create_task(output_handler()) diff --git a/vllm_omni/entrypoints/omni.py b/vllm_omni/entrypoints/omni.py index 3d918aea7..e76fe834a 100644 --- a/vllm_omni/entrypoints/omni.py +++ b/vllm_omni/entrypoints/omni.py @@ -232,6 +232,8 @@ class OmniBase: self.worker_backend = worker_backend self.ray_address = ray_address self.batch_timeout = batch_timeout + # async chunk remains the same for each stage + self.async_chunk = self._is_async_chunk_enable(self.stage_configs) # Build OmniStage instances in parallel, preserve original order def _build_stage(idx_cfg: tuple[int, Any]) -> tuple[int, OmniStage]: @@ -261,6 +263,11 @@ class OmniBase: # Wait for all stages to report readiness before seeding self._wait_for_stages_ready(timeout=init_timeout) + def _is_async_chunk_enable(self, stage_args: list) -> bool: + """get async chunk flag""" + engine_args = getattr(stage_args[0], "engine_args", None) + return bool(getattr(engine_args, "async_chunk", False)) + def _start_stages(self, model: str) -> None: """Start all stage processes.""" if self.worker_backend == "ray": diff --git a/vllm_omni/entrypoints/omni_stage.py b/vllm_omni/entrypoints/omni_stage.py index 7beef030d..e0e699c9c 100644 --- a/vllm_omni/entrypoints/omni_stage.py +++ b/vllm_omni/entrypoints/omni_stage.py @@ -673,12 +673,15 @@ def _stage_worker( e, ) # Init engine based on stage_type - logger.debug( - "[Stage-%s] Initializing %s engine with args keys=%s", - stage_id, - stage_type, - list(engine_args.keys()), - ) + logger.debug("[Stage-%s] Initializing %s engine with args keys=%s", stage_id, stage_type, list(engine_args.keys())) + if engine_args.get("async_chunk", False): + logger.debug("[Stage-%s] Async chunk enabled, injecting connectors config", stage_id) + stage_connector_spec = {} + for v in connectors_config.values(): + stage_connector_spec = dict(v.get("spec", {})) + break + engine_args["stage_connector_spec"] = stage_connector_spec + engine_args["stage_id"] = stage_id try: if stage_type == "diffusion": engine_args.pop("model_stage", None) @@ -1206,6 +1209,14 @@ async def _stage_worker_async( stage_type, list(engine_args.keys()), ) + if engine_args.get("async_chunk", False): + logger.debug("[Stage-%s] Async chunk enabled, injecting connectors config", stage_id) + stage_connector_spec = {} + for v in connectors_config.values(): + stage_connector_spec = dict(v.get("spec", {})) + break + engine_args["stage_connector_spec"] = stage_connector_spec + engine_args["stage_id"] = stage_id try: if stage_type == "diffusion": # For diffusion, we need to extract diffusion-specific config diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 8a62fd16c..5d9f76d70 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -18,6 +18,7 @@ try: except ImportError: soundfile = None + from openai.types.chat.chat_completion_audio import ChatCompletionAudio as OpenAIChatCompletionAudio from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, @@ -1213,6 +1214,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): yield f"data: {data}\n\n" elif final_output_type == "audio": + role = self.get_chat_request_role(request) choices_data = self._create_audio_choice(omni_res, role, request, stream=True) chunk = OmniChatCompletionStreamResponse( id=request_id, @@ -1655,7 +1657,10 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): ): choices: list[ChatCompletionResponseChoice] = [] final_res = omni_outputs.request_output - audio_tensor = final_res.multimodal_output["audio"].float().detach().cpu().numpy() + if stream: + audio_tensor = final_res.multimodal_output["audio"][-1].float().detach().cpu().numpy() + else: + audio_tensor = final_res.multimodal_output["audio"].float().detach().cpu().numpy() # Ensure audio is 1D (flatten if needed) if audio_tensor.ndim > 1: diff --git a/vllm_omni/entrypoints/stage_utils.py b/vllm_omni/entrypoints/stage_utils.py index db17116ae..da837ce9a 100644 --- a/vllm_omni/entrypoints/stage_utils.py +++ b/vllm_omni/entrypoints/stage_utils.py @@ -222,7 +222,8 @@ def maybe_dump_to_shm(obj: Any, threshold: int) -> tuple[bool, Any]: """ payload = serialize_obj(obj) if len(payload) > threshold: - return True, shm_write_bytes(payload) + logger.debug(f"Dumping object to SHM with size: {len(payload)}") + return True, shm_write_bytes(payload, name=None) return False, obj diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index bbb75a968..f1ed4d843 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -221,6 +221,7 @@ def load_stage_configs_from_yaml(config_path: str, base_engine_args: dict | None base_engine_args = {} config_data = OmegaConf.load(config_path) stage_args = config_data.stage_args + global_async_chunk = config_data.get("async_chunk", False) # Convert any nested dataclass objects to dicts before creating OmegaConf base_engine_args = _convert_dataclasses_to_dict(base_engine_args) base_engine_args = OmegaConf.create(base_engine_args) @@ -234,6 +235,7 @@ def load_stage_configs_from_yaml(config_path: str, base_engine_args: dict | None runtime_cfg = stage_arg.runtime max_batch_size = int(runtime_cfg.get("max_batch_size", 1) or 1) base_engine_args_tmp["max_num_seqs"] = max_batch_size + base_engine_args_tmp.async_chunk = global_async_chunk stage_arg.engine_args = base_engine_args_tmp return stage_args diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py index 17f16d1d9..0dd590710 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py @@ -180,6 +180,8 @@ class Qwen3OmniMoeForConditionalGeneration( self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None ) + self.chunk_segment_info = {} # request_id -> chunk_segment_info + # ==================== Device utilities ==================== @staticmethod @@ -483,12 +485,19 @@ class Qwen3OmniMoeForConditionalGeneration( talker_codes = talker_codes.unsqueeze(0).unsqueeze(0) talker_codes = talker_codes.expand(1, 16, -1) - # Use chunked decode for memory efficiency - audio_tensor = self.code2wav.chunked_decode( - talker_codes, - chunk_size=300, - left_context_size=25, - ) + if self.vllm_config.model_config.async_chunk: + audio_tensor = self.code2wav.chunked_decode_streaming( + talker_codes, + chunk_size=25, + left_context_size=25, + ) + else: + # Use chunked decode for memory efficiency + audio_tensor = self.code2wav.chunked_decode( + talker_codes, + chunk_size=300, + left_context_size=25, + ) return audio_tensor @@ -595,7 +604,6 @@ class Qwen3OmniMoeForConditionalGeneration( last_talker_hidden, text_step, update_dict = self.talker_preprocess_decode( input_ids, input_embeds, **info_dict ) - update_dict["mtp_inputs"] = last_talker_hidden, text_step return input_ids, input_embeds, update_dict @@ -624,6 +632,33 @@ class Qwen3OmniMoeForConditionalGeneration( inputs_embeds = (inputs_embeds + text_step).reshape(-1, self.talker_config.text_config.hidden_size) return inputs_embeds, code_predictor_codes.squeeze(-1) + def _get_tts_embed(self, thinker_embed, tts_bos_thinker, tts_eos_thinker, tts_pad_thinker): + """Project thinker-side TTS embeddings into talker text space.""" + module_device = self._module_device(self.talker) + + def _ensure_1x1(x: torch.Tensor) -> torch.Tensor: + if x.ndim == 3: + return x[0, -1:, :] + if x.ndim == 2: + return x[-1] + return x.view(1, 1, -1) + + def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor: + if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0: + xin = _ensure_1x1(x_opt).to(module_device) + else: + xin = torch.zeros( + (1, thinker_embed.shape[-1]), + device=module_device, + dtype=thinker_embed.dtype, + ) + return self.talker.text_projection(xin).to(module_device) + + self.tts_bos_embed = _proj_from_thinker(tts_bos_thinker) + self.tts_eos_embed = _proj_from_thinker(tts_eos_thinker) + self.tts_pad_embed = _proj_from_thinker(tts_pad_thinker) + return self.tts_bos_embed, self.tts_eos_embed, self.tts_pad_embed + def talker_preprocess_prefill(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict): # Containers to return per-request updates (e.g., code_predictor_hidden_per_request) update_dict: dict[str, dict] = {} @@ -647,7 +682,7 @@ class Qwen3OmniMoeForConditionalGeneration( if info_dict.get("thinker_input_ids") is None else torch.as_tensor(info_dict.get("thinker_input_ids"), device=self._module_device(self.talker)) ) - # Optional thinker-side TTS token embeddings (to be projected by talker) + tts_bos_thinker = info_dict.get("tts_bos_embed").to( device=self._module_device(self.talker), dtype=torch.bfloat16 ) @@ -700,7 +735,7 @@ class Qwen3OmniMoeForConditionalGeneration( tts_pad_thinker=tts_pad_thinker, ) - # Queue tailing_text_hidden for decode (drop first for next steps), + # Queue trailing_text_hidden for decode (drop first for next steps), try: if isinstance(trailing_text_hidden, torch.Tensor) and trailing_text_hidden.numel() > 0: if trailing_text_hidden.ndim == 2: @@ -716,7 +751,7 @@ class Qwen3OmniMoeForConditionalGeneration( # compatible with old shape [1,S,D] rem_tail = trailing_text_hidden.squeeze(0) if rem_tail.shape[0] > 0: - update_dict["tailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous() + update_dict["trailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous() # Also persist projected tts_pad for decode fallback if needed if isinstance(tts_pad_thinker, torch.Tensor): pad_in = tts_pad_thinker @@ -762,31 +797,9 @@ class Qwen3OmniMoeForConditionalGeneration( (thinker_result_ids == self.thinker_config.video_token_id) ).to(input_ids.device) # [t] # fmt: skip - # Project thinker-side TTS embeddings into talker text space (provided by thinker stage) - talker_dev = self._module_device(self.talker) - - def _ensure_1x1(x: torch.Tensor) -> torch.Tensor: - if x.ndim == 3: - return x[0, -1:, :] - if x.ndim == 2: - return x[-1] - return x.view(1, 1, -1) - - def _proj_from_thinker(x_opt: torch.Tensor | None) -> torch.Tensor: - if isinstance(x_opt, torch.Tensor) and x_opt.numel() > 0: - xin = _ensure_1x1(x_opt).to(talker_dev) - else: - xin = torch.zeros( - (1, thinker_embed.shape[-1]), - device=talker_dev, - dtype=thinker_embed.dtype, - ) - return self.talker.text_projection(xin).to(input_ids.device) - - tts_bos_embed = _proj_from_thinker(tts_bos_thinker) - tts_eos_embed = _proj_from_thinker(tts_eos_thinker) - tts_pad_embed = _proj_from_thinker(tts_pad_thinker) - self.tts_pad_embed = tts_pad_embed + tts_bos_embed, tts_eos_embed, tts_pad_embed = self._get_tts_embed( + thinker_embed, tts_bos_thinker, tts_eos_thinker, tts_pad_thinker + ) talker_input_embeds = [] # [1 t d] talker_input_ids = [] @@ -835,24 +848,58 @@ class Qwen3OmniMoeForConditionalGeneration( return talker_input_id, talker_input_embed, trailing_text_hidden_all + def _thinker_decode_to_talker_decode( + self, + info_dict: dict, + device: torch.device, + update_dict, + ): + """ + Project thinker outputs to talker inputs during prefill stage. + Returns: + (input_ids, input_embeds) for talker + """ + thinker_embed = info_dict.get("thinker_embeddings", None) + if thinker_embed is None: + if info_dict.get("finished_flag"): + return self.tts_pad_embed.to(device) + update_dict["finished_flag"] = True + return self.tts_eos_embed.to(device) + + thinker_embed = thinker_embed.to(device) + return self.talker.text_projection(thinker_embed).to(device) + def talker_preprocess_decode(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict): update_dict: dict[str, dict] = {} + last_talker_hidden = None + text_step = None try: - q_tail = info_dict.get("tailing_text_hidden") - if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0: - use_vec = q_tail[0:1, :] - new_q_tail = ( - q_tail[1:, :].detach().to("cpu").contiguous() - if q_tail.shape[0] > 1 - else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype) - ) - text_step = use_vec.to(input_embeds.device, dtype=input_embeds.dtype) - update_dict["tailing_text_hidden"] = new_q_tail + if self.vllm_config.model_config.async_chunk: + text_step = self._thinker_decode_to_talker_decode(info_dict, input_ids.device, update_dict) else: - text_step = self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype) + q_tail = info_dict.get("trailing_text_hidden", None) + if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0: + use_vec = q_tail[0:1, :] + new_q_tail = ( + q_tail[1:, :].detach().to("cpu").contiguous() + if q_tail.shape[0] > 1 + else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype) + ) + text_step = use_vec.to(input_embeds.device, dtype=input_embeds.dtype) + update_dict["trailing_text_hidden"] = new_q_tail + else: + text_step = self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype) - last_talker_hidden = info_dict.get("last_talker_hidden").to(input_embeds.device, dtype=input_embeds.dtype) - last_talker_hidden = last_talker_hidden.reshape(*last_talker_hidden.shape[-2:]) # [1, hidden_size] + last_talker_hidden_tensor = info_dict.get("last_talker_hidden") + if last_talker_hidden_tensor is not None: + last_talker_hidden = last_talker_hidden_tensor.to(input_embeds.device, dtype=input_embeds.dtype) + last_talker_hidden = last_talker_hidden.reshape(*last_talker_hidden.shape[-2:]) # [1, hidden_size] + else: + last_talker_hidden = torch.zeros( + (1, self.talker_config.text_config.hidden_size), + device=input_embeds.device, + dtype=input_embeds.dtype, + ) except Exception as e: logger.error(f"Error in decode: {e}") @@ -866,7 +913,6 @@ class Qwen3OmniMoeForConditionalGeneration( ) user_mm_mask = multimodal_mask[im_start_index:segment_end_index] - # Multimodal data exists if user_mm_mask.any(): user_thinker_hidden_mm = thinker_hidden[im_start_index:segment_end_index][user_mm_mask] @@ -883,12 +929,20 @@ class Qwen3OmniMoeForConditionalGeneration( assistant_hidden = self.talker.text_projection(thinker_embed[im_start_index:segment_end_index]).to( tts_pad_embed.device ) # [t, d] + + # [3 tokens] + [4 pad] + [1 BOS] + [1 first text] = 9 tokens assistant_text_hidden = torch.cat( ( assistant_hidden[:3], tts_pad_embed.expand(4, -1), tts_bos_embed, - assistant_hidden[3:4], # First text + assistant_hidden[3:4] + if assistant_hidden.shape[0] > 3 + else torch.zeros( + (1, assistant_hidden.shape[1]), + device=assistant_hidden.device, + dtype=assistant_hidden.dtype, + ), # First text ), dim=0, ) @@ -904,6 +958,9 @@ class Qwen3OmniMoeForConditionalGeneration( device=tts_pad_embed.device, dtype=torch.long, ) + embed_input_ids = self.talker.embed_input_ids(codec_special_tokens).to( + device=tts_pad_embed.device, dtype=torch.bfloat16 + ) assistant_codec_hidden = torch.cat( ( torch.zeros( @@ -911,18 +968,21 @@ class Qwen3OmniMoeForConditionalGeneration( device=tts_pad_embed.device, dtype=torch.bfloat16, ), - self.talker.embed_input_ids(codec_special_tokens).to(device=tts_pad_embed.device, dtype=torch.bfloat16), - ), - dim=0, - ) - trailing_text_hidden = torch.cat( - ( - assistant_hidden[4:], - tts_eos_embed, + embed_input_ids, ), dim=0, ) + if assistant_hidden.shape[0] > 4: + trailing_text_hidden = torch.cat( + (assistant_hidden[4:], tts_eos_embed), + dim=0, + ) + else: + trailing_text_hidden = torch.zeros( + tts_eos_embed.shape, device=tts_eos_embed.device, dtype=tts_eos_embed.dtype + ) + input_embeds = assistant_text_hidden + assistant_codec_hidden input_ids = torch.full( (assistant_text_hidden.shape[0],), diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py index f430ecb16..bc2242f70 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py @@ -196,6 +196,39 @@ class Qwen3OmniMoeCode2Wav(nn.Module): return torch.cat(wavs, dim=-1) + def chunked_decode_streaming( + self, + codes: torch.Tensor, + chunk_size: int = 25, + left_context_size: int = 25, + ) -> torch.Tensor: + """ + Decode long sequences in chunks to avoid OOM. + + Uses overlapping chunks with left context to avoid boundary artifacts. + + Args: + codes: [batch, num_quantizers, seq_len] - num_quantizers-layer RVQ codes + chunk_size: Number of codec frames per chunk + left_context_size: Number of overlapping frames for context + + Returns: + waveform: [batch, 1, waveform_len] - Complete waveform + """ + wavs = [] + end_index = codes.shape[-1] + # TODO: need to optimize algorithms, current only support + # chunk_size = left_context_size = 25 + if end_index == chunk_size: + context_size = 0 + else: + context_size = left_context_size + # Decode chunk + wav_chunk = self(codes) + # Remove context from output (context_size * total_upsample samples) + wavs.append(wav_chunk[..., context_size * self.total_upsample :]) + return torch.cat(wavs, dim=-1) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load weights from HuggingFace checkpoint.""" loader = AutoWeightsLoader( diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml index d4de07823..9c422d76e 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml @@ -4,12 +4,13 @@ # Stage 2: Code2Wav (8-layer RVQ codes → audio waveform) # The following config has been verified on 2x H100-80G GPUs. +async_chunk: false stage_args: - stage_id: 0 stage_type: llm # Use llm stage type to launch OmniLLM runtime: devices: "0,1" - max_batch_size: 1 + max_batch_size: 64 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration @@ -40,17 +41,16 @@ stage_args: stage_type: llm # Use llm stage type to launch OmniLLM runtime: devices: "1" - max_batch_size: 1 + max_batch_size: 64 engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.35 + gpu_memory_utilization: 0.3 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav - # tensor_parallel_size: 2 enable_prefix_caching: false max_num_batched_tokens: 32768 distributed_executor_backend: "mp" diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml new file mode 100644 index 000000000..55d455f56 --- /dev/null +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml @@ -0,0 +1,101 @@ +# Stage config for running Qwen3-Omni-MoE with 3-stage architecture +# Stage 0: Thinker (multimodal understanding + text generation) +# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes) +# Stage 2: Code2Wav (16-layer RVQ codes → audio waveform) + +# The following config has been verified on 2x H100-80G GPUs. +async_chunk: true +stage_args: + - stage_id: 0 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + devices: "0,1" + max_batch_size: 64 + engine_args: + model_stage: thinker + model_arch: Qwen3OmniMoeForConditionalGeneration + worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + gpu_memory_utilization: 0.6 + enforce_eager: false + trust_remote_code: true + engine_output_type: latent # Output hidden states for talker + distributed_executor_backend: "mp" + enable_prefix_caching: false + max_num_batched_tokens: 32768 + hf_config_name: thinker_config + tensor_parallel_size: 2 + custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk + final_output: true + final_output_type: text + is_comprehension: true + default_sampling_params: + temperature: 0.4 + top_p: 0.9 + top_k: 1 + max_tokens: 2048 + seed: 42 + detokenize: True + repetition_penalty: 1.05 + + - stage_id: 1 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + devices: "0" + max_batch_size: 64 + engine_args: + model_stage: talker + model_arch: Qwen3OmniMoeForConditionalGeneration + worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + gpu_memory_utilization: 0.3 + enforce_eager: false + trust_remote_code: true + engine_output_type: latent # Output codec codes for code2wav + enable_prefix_caching: false + max_num_batched_tokens: 32768 + distributed_executor_backend: "mp" + hf_config_name: talker_config + custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk + engine_input_source: [0] + # final_output: true + # final_output_type: text + default_sampling_params: + temperature: 0.9 + top_k: 50 + max_tokens: 4096 + seed: 42 + detokenize: False + repetition_penalty: 1.05 + stop_token_ids: [2150] + + - stage_id: 2 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + devices: "1" + max_batch_size: 1 + engine_args: + model_stage: code2wav + model_arch: Qwen3OmniMoeForConditionalGeneration + worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler + enforce_eager: true + trust_remote_code: true + async_scheduling: false + enable_prefix_caching: false + engine_output_type: audio # Final output: audio waveform + gpu_memory_utilization: 0.1 + distributed_executor_backend: "mp" + max_num_batched_tokens: 10000 + hf_config_name: thinker_config + engine_input_source: [1] + final_output: true + final_output_type: audio + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + seed: 42 + detokenize: True + repetition_penalty: 1.1 diff --git a/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py b/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py index a1457a975..03daa8e42 100644 --- a/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py +++ b/vllm_omni/model_executor/stage_input_processors/qwen3_omni.py @@ -9,6 +9,7 @@ import torch from vllm.inputs import TextPrompt from vllm.platforms import current_platform +from vllm_omni.engine import OmniEngineCoreRequest from vllm_omni.inputs.data import OmniTokensPrompt @@ -48,6 +49,74 @@ def _compute_talker_prompt_ids_length(info, device: torch.device | str = "cuda") return sum_user_len + assistant_len +# ========================= +# Common helpers +# ========================= + + +def _ensure_list(x): + """Convert ConstantList / tensor-like to Python list.""" + if hasattr(x, "_x"): + return list(x._x) + elif not isinstance(x, list): + return x + return list(x) + + +def _validate_stage_inputs(stage_list, engine_input_source): + if not engine_input_source: + raise ValueError("engine_input_source cannot be empty") + + stage_id = engine_input_source[0] + if stage_id >= len(stage_list): + raise IndexError(f"Invalid stage_id: {stage_id}") + + stage = stage_list[stage_id] + if stage.engine_outputs is None: + raise RuntimeError(f"Stage {stage_id} has no outputs yet") + + return stage.engine_outputs + + +# ========================= +# Thinker -> Talker +# ========================= + + +def thinker2talker_async_chunk( + pooling_output: dict[str, Any], + request: OmniEngineCoreRequest, +) -> list[dict[str, Any]]: + """ + Process thinker outputs to create talker inputs. + 1. thinker's text generation outputs (token IDs + hidden states) + 2. Split hidden states into: prompt embeddings + generated embeddings + 3. Package for talker with additional information + """ + all_token_ids = request.all_token_ids # prefill + decode + prompt_token_ids = request.prompt_token_ids + + # Convert ConstantList to regular list for OmniSerializer serialization + all_token_ids = _ensure_list(all_token_ids) + prompt_token_ids = _ensure_list(prompt_token_ids) + + thinker_output = pooling_output + + talker_additional_info = { + "thinker_embeddings": thinker_output.get("0").detach().cpu(), + "thinker_hidden_states": thinker_output.get("24").detach().cpu(), + "thinker_sequences": all_token_ids, + "thinker_input_ids": prompt_token_ids, + # Provide thinker-side TTS token embeddings for talker projection + "tts_bos_embed": thinker_output.get("tts_bos_embed").detach().cpu(), + "tts_eos_embed": thinker_output.get("tts_eos_embed").detach().cpu(), + "tts_pad_embed": thinker_output.get("tts_pad_embed").detach().cpu(), + "finished": torch.tensor(request.is_finished(), dtype=torch.bool), + } + + return talker_additional_info + + def thinker2talker( stage_list: list[Any], engine_input_source: list[int], @@ -71,47 +140,33 @@ def thinker2talker( Returns: List of OmniTokensPrompt for talker stage """ - if not engine_input_source: - raise ValueError("engine_input_source cannot be empty") - - source_stage_id = engine_input_source[0] - if source_stage_id >= len(stage_list): - raise IndexError(f"Invalid stage_id: {source_stage_id}") - - if stage_list[source_stage_id].engine_outputs is None: - raise RuntimeError(f"Stage {source_stage_id} has no outputs yet") - - thinker_outputs = stage_list[source_stage_id].engine_outputs - talker_inputs = [] + thinker_outputs = _validate_stage_inputs(stage_list, engine_input_source) + talker_inputs: list[OmniTokensPrompt] = [] device = torch.device(current_platform.device_type) # Process each thinker output - for i, thinker_output in enumerate(thinker_outputs): + for thinker_output in thinker_outputs: output = thinker_output.outputs[0] - thinker_embeddings = output.multimodal_output["0"].detach().to(device=device, dtype=torch.float) - thinker_hidden_states = output.multimodal_output["24"].detach().to(device=device, dtype=torch.float) info = { - "thinker_embeddings": thinker_embeddings, - "thinker_hidden_states": thinker_hidden_states, - "thinker_sequences": thinker_output.prompt_token_ids - + output.token_ids, # the thinker_sequences is the whole ids + "thinker_embeddings": output.multimodal_output["0"].detach().to(device=device, dtype=torch.float), + "thinker_hidden_states": output.multimodal_output["24"].detach().to(device=device, dtype=torch.float), + "thinker_sequences": ( + thinker_output.prompt_token_ids + output.token_ids + ), # the thinker_sequences is the whole ids "thinker_input_ids": thinker_output.prompt_token_ids, # Provide thinker-side TTS token embeddings for talker projection - "tts_bos_embed": ( - output.multimodal_output.get("tts_bos_embed").detach().to(device=device, dtype=torch.float) - ), - "tts_eos_embed": ( - output.multimodal_output.get("tts_eos_embed").detach().to(device=device, dtype=torch.float) - ), - "tts_pad_embed": ( - output.multimodal_output.get("tts_pad_embed").detach().to(device=device, dtype=torch.float) - ), + "tts_bos_embed": output.multimodal_output["tts_bos_embed"].detach().to(device=device, dtype=torch.float), + "tts_eos_embed": output.multimodal_output["tts_eos_embed"].detach().to(device=device, dtype=torch.float), + "tts_pad_embed": output.multimodal_output["tts_pad_embed"].detach().to(device=device, dtype=torch.float), } + + prompt_len = _compute_talker_prompt_ids_length(info, device=device) + talker_inputs.append( OmniTokensPrompt( - prompt_token_ids=[0] * _compute_talker_prompt_ids_length(info, device=device), + prompt_token_ids=[0] * prompt_len, additional_information=info, multi_modal_data=None, mm_processor_kwargs=None, @@ -121,6 +176,50 @@ def thinker2talker( return talker_inputs +# ========================= +# Talker -> Code2Wav +# ========================= + + +def talker2code2wav_async_chunk( + pooling_output: dict[str, Any], + request: OmniEngineCoreRequest, +): + """ + Pooling version. + """ + if "code_predictor_codes" not in pooling_output: + return [] + + code_predictor_codes = pooling_output["code_predictor_codes"] + + if code_predictor_codes is None: + return [] + if isinstance(code_predictor_codes, torch.Tensor): + if code_predictor_codes.numel() == 0: + return [] + elif hasattr(code_predictor_codes, "__len__"): + if len(code_predictor_codes) == 0: + return [] + + if isinstance(code_predictor_codes, torch.Tensor): + if not code_predictor_codes.any(): + return [] + else: + code_tensor = torch.tensor(code_predictor_codes, dtype=torch.long) + if not code_tensor.any(): + return [] + + codec_codes = code_predictor_codes.to(torch.long).transpose(0, 1).cpu().to(torch.long).reshape(-1).tolist() + if sum(codec_codes) == 0: + return [] + + return { + "code_predictor_codes": codec_codes, + "finished": torch.tensor(request.is_finished(), dtype=torch.bool), + } + + def talker2code2wav( stage_list: list[Any], engine_input_source: list[int], @@ -144,21 +243,10 @@ def talker2code2wav( Returns: List of OmniTokensPrompt for code2wav stage """ - if not engine_input_source: - raise ValueError("engine_input_source cannot be empty") - - source_stage_id = engine_input_source[0] - if source_stage_id >= len(stage_list): - raise IndexError(f"Invalid stage_id: {source_stage_id}") - - if stage_list[source_stage_id].engine_outputs is None: - raise RuntimeError(f"Stage {source_stage_id} has no outputs yet") - - talker_outputs = stage_list[source_stage_id].engine_outputs - code2wav_inputs = [] - + talker_outputs = _validate_stage_inputs(stage_list, engine_input_source) + code2wav_inputs: list[OmniTokensPrompt] = [] # Process each talker output - for i, talker_output in enumerate(talker_outputs): + for talker_output in talker_outputs: output = talker_output.outputs[0] seq_len = len(output.token_ids) - 1 # Extract codec codes from talker output diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index 6055c93f7..ecc839fd3 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -280,7 +280,6 @@ class GPUARModelRunner(OmniGPUModelRunner): aux_hidden_states = None hidden_states, multimodal_outputs = self.extract_multimodal_outputs(model_output) - if multimodal_outputs is not None: keys_or_type = ( list(multimodal_outputs.keys()) @@ -501,7 +500,9 @@ class GPUARModelRunner(OmniGPUModelRunner): dtype=np.int32, ) - self._process_additional_information_updates(hidden_states, multimodal_outputs, num_scheduled_tokens_np) + self._process_additional_information_updates( + hidden_states, multimodal_outputs, num_scheduled_tokens_np, scheduler_output + ) pooler_output: list[dict[str, object]] = [] for rid in req_ids_output_copy: @@ -761,8 +762,9 @@ class GPUARModelRunner(OmniGPUModelRunner): for attempt in range(max_retries): try: + put_key = f"omni_{from_stage}_to_{to_stage}_{request_id}" success, size, metadata = connector.put( - from_stage=from_stage, to_stage=to_stage, request_id=request_id, data=data + from_stage=from_stage, to_stage=to_stage, put_key=put_key, data=data ) if success: return success, size, metadata diff --git a/vllm_omni/worker/gpu_generation_model_runner.py b/vllm_omni/worker/gpu_generation_model_runner.py index 7d5802483..87e1419c8 100644 --- a/vllm_omni/worker/gpu_generation_model_runner.py +++ b/vllm_omni/worker/gpu_generation_model_runner.py @@ -50,6 +50,17 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): - Executes generation process and returns tensors via `pooler_output`. """ + def _update_request_states(self, scheduler_output: SchedulerOutput): + cached_reqs = scheduler_output.scheduled_cached_reqs + for _, req_id in enumerate(cached_reqs.req_ids): + req_state = self.requests.get(req_id) + assert req_state is not None + req_state.prompt_token_ids = cached_reqs.prompt_token_ids.get(req_id) + self.input_batch.remove_request(req_id) + # update the request state in self.input_batch + self.input_batch.add_request(req_state) + self._init_mrope_positions(req_state) + @torch.inference_mode() def execute_model( self, @@ -74,6 +85,8 @@ class GPUGenerationModelRunner(OmniGPUModelRunner): record_function_or_nullcontext("gpu_model_runner: preprocess"), self.synchronize_input_prep(), ): + if self.model_config.async_chunk: + self._update_request_states(scheduler_output) self._update_states(scheduler_output) if not scheduler_output.total_num_scheduled_tokens: return EMPTY_MODEL_RUNNER_OUTPUT diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index 06abd9f3a..a486212aa 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -808,6 +808,7 @@ class OmniGPUModelRunner(GPUModelRunner): hidden_states: torch.Tensor, multimodal_outputs: object, num_scheduled_tokens_np: np.ndarray, + scheduler_output: "SchedulerOutput", ) -> None: """Process model-provided per-request additional_information updates and merge into request state.""" try: @@ -815,10 +816,13 @@ class OmniGPUModelRunner(GPUModelRunner): # TODO(Peiqi): do we have a more elegant way to do this? if hasattr(self.model, "has_postprocess") and self.model.has_postprocess: for req_index, req_id in enumerate(self.input_batch.req_ids): - req_state = self.requests.get(req_id) - req_infos = ( - getattr(req_state, "additional_information_cpu", None) if req_state is not None else None - ) + if self.model_config.async_chunk: + req_infos = self._get_additional_information(scheduler_output, req_id) + else: + req_state = self.requests.get(req_id) + req_infos = ( + getattr(req_state, "additional_information_cpu", None) if req_state is not None else None + ) start_offset = int(self.query_start_loc.cpu[req_index]) sched_tokens = int(num_scheduled_tokens_np[req_index]) s, e = start_offset, start_offset + sched_tokens @@ -859,6 +863,40 @@ class OmniGPUModelRunner(GPUModelRunner): start_offset = int(self.query_start_loc.cpu[req_index]) self.inputs_embeds[start_offset : start_offset + overlay_len].copy_(src) + def _get_additional_information(self, scheduler_output: "SchedulerOutput", req_id: str) -> dict: + req_infos = None + req_state = self.requests.get(req_id) + additional_information_cpu = getattr(req_state, "additional_information_cpu", None) + for new_req in scheduler_output.scheduled_new_reqs: + if new_req.req_id == req_id: + payload_info = getattr(new_req, "additional_information", None) + if payload_info is not None: + return payload_info + + if hasattr(scheduler_output.scheduled_cached_reqs, "additional_information"): + cached_infos = getattr(scheduler_output.scheduled_cached_reqs, "additional_information", {}) + if isinstance(cached_infos, dict) and req_id in cached_infos: + req_infos = cached_infos[req_id] + if not isinstance(req_infos, dict): + req_infos = None + + if req_infos is None or req_infos.get("last_talker_hidden", None) is None: + if req_infos is None: + additional_information_cpu.pop("thinker_embeddings", None) + req_infos = additional_information_cpu + else: + req_infos["last_talker_hidden"] = additional_information_cpu.get("last_talker_hidden", None) + req_infos["num_processed_thinker_tokens"] = additional_information_cpu.get( + "num_processed_thinker_tokens", 0 + ) + if not isinstance(req_infos, dict): + req_infos = None + + if req_infos is None: + logger.warning(f"No additional_information found for req_id: {req_id}") + + return req_infos + def _preprocess( self, scheduler_output: "SchedulerOutput", @@ -974,9 +1012,14 @@ class OmniGPUModelRunner(GPUModelRunner): # collect additional_information (tensor/list) for prefill portion only decode_req_ids = [] for req_index, req_id in enumerate(self.input_batch.req_ids): - req_state = self.requests.get(req_id) - req_infos = getattr(req_state, "additional_information_cpu", None) if req_state is not None else None - + # Try to get additional_information from multiple sources + if self.vllm_config.model_config.async_chunk: + req_infos = self._get_additional_information(scheduler_output, req_id) + else: + req_state = self.requests.get(req_id) + req_infos = ( + getattr(req_state, "additional_information_cpu", None) if req_state is not None else None + ) start_offset = int(self.query_start_loc.cpu[req_index]) sched_tokens = int(num_scheduled_tokens_np[req_index]) s, e = start_offset, start_offset + sched_tokens From 978a290d6e26da93ac035624ce7d7ee7f3e8b132 Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Mon, 26 Jan 2026 13:09:03 +0800 Subject: [PATCH 16/83] [Bugfix] Fix diffusion pipeline CFG (guidance_scale parsing failure bug) (#922) Signed-off-by: samithuang <285365963@qq.com> Signed-off-by: Hongsheng Liu Co-authored-by: Hongsheng Liu --- .../text_to_video/text_to_video.md | 7 ++++--- .../models/qwen_image/pipeline_qwen_image.py | 2 ++ .../qwen_image/pipeline_qwen_image_edit.py | 2 ++ .../pipeline_qwen_image_edit_plus.py | 2 ++ .../qwen_image/pipeline_qwen_image_layered.py | 2 ++ .../models/wan2_2/pipeline_wan2_2.py | 20 +++++++++++-------- .../models/wan2_2/pipeline_wan2_2_i2v.py | 20 +++++++++++-------- .../models/wan2_2/pipeline_wan2_2_ti2v.py | 20 +++++++++++-------- vllm_omni/entrypoints/async_omni_diffusion.py | 3 +++ 9 files changed, 51 insertions(+), 27 deletions(-) diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index f0ce5495a..7204ec58a 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -9,10 +9,11 @@ python text_to_video.py \ --prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \ --negative_prompt "" \ --height 480 \ - --width 640 \ + --width 832 \ --num_frames 33 \ --guidance_scale 4.0 \ --guidance_scale_high 3.0 \ + --flow_shift 12.0 \ --num_inference_steps 40 \ --fps 16 \ --output t2v_out.mp4 @@ -21,9 +22,9 @@ python text_to_video.py \ Key arguments: - `--prompt`: text description (string). -- `--height/--width`: output resolution (defaults 720x1280). Dimensions should align with Wan VAE downsampling (multiples of 8). +- `--height/--width`: output resolution (defaults 480x832, i.e. 480P). Dimensions should align with Wan VAE downsampling (multiples of 8). - `--num_frames`: Number of frames (Wan default is 81). -- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high).. +- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high). - `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string). - `--boundary_ratio`: Boundary split ratio for low/high DiT. - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video). diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py index 5ae8228a0..2b9d906d3 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py @@ -672,6 +672,8 @@ class QwenImagePipeline( num_inference_steps = req.num_inference_steps or num_inference_steps generator = req.generator or generator true_cfg_scale = req.true_cfg_scale or true_cfg_scale + if req.guidance_scale_provided: + guidance_scale = req.guidance_scale req_num_outputs = getattr(req, "num_outputs_per_prompt", None) if req_num_outputs and req_num_outputs > 0: num_images_per_prompt = req_num_outputs diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py index 4927ca3ac..ac74507c7 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py @@ -748,6 +748,8 @@ class QwenImageEditPipeline( num_inference_steps = req.num_inference_steps or num_inference_steps generator = req.generator or generator true_cfg_scale = req.true_cfg_scale or true_cfg_scale + if req.guidance_scale_provided: + guidance_scale = req.guidance_scale req_num_outputs = getattr(req, "num_outputs_per_prompt", None) if req_num_outputs and req_num_outputs > 0: num_images_per_prompt = req_num_outputs diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py index 9eeaed5f9..4e2fe35f1 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py @@ -704,6 +704,8 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): num_inference_steps = req.num_inference_steps or num_inference_steps generator = req.generator or generator true_cfg_scale = req.true_cfg_scale or true_cfg_scale + if req.guidance_scale_provided: + guidance_scale = req.guidance_scale req_num_outputs = getattr(req, "num_outputs_per_prompt", None) if req_num_outputs and req_num_outputs > 0: num_images_per_prompt = req_num_outputs diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py index 4642b3eb4..ae2d039e3 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py @@ -722,6 +722,8 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as num_inference_steps = req.num_inference_steps or num_inference_steps generator = req.generator or generator true_cfg_scale = req.true_cfg_scale or true_cfg_scale + if req.guidance_scale_provided: + guidance_scale = req.guidance_scale req_num_outputs = getattr(req, "num_outputs_per_prompt", None) if req_num_outputs and req_num_outputs > 0: num_images_per_prompt = req_num_outputs diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index 3b04e2a5a..8ff6271a0 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -284,11 +284,11 @@ class Wan22Pipeline(nn.Module): req: OmniDiffusionRequest, prompt: str | None = None, negative_prompt: str | None = None, - height: int | None = None, - width: int | None = None, - num_inference_steps: int | None = None, + height: int = 480, + width: int = 832, + num_inference_steps: int = 40, guidance_scale: float | tuple[float, float] = 4.0, - frame_num: int | None = None, + frame_num: int = 81, output_type: str | None = "np", generator: torch.Generator | None = None, prompt_embeds: torch.Tensor | None = None, @@ -301,9 +301,9 @@ class Wan22Pipeline(nn.Module): if prompt is None and prompt_embeds is None: raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.") - height = req.height or height or 720 - width = req.width or width or 1280 - num_frames = req.num_frames if req.num_frames else frame_num or 81 + height = req.height or height + width = req.width or width + num_frames = req.num_frames if req.num_frames else frame_num # Ensure dimensions are compatible with VAE and patch size # For expand_timesteps mode, we need latent dims to be even (divisible by patch_size) @@ -311,7 +311,11 @@ class Wan22Pipeline(nn.Module): mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V height = (height // mod_value) * mod_value width = (width // mod_value) * mod_value - num_steps = req.num_inference_steps or num_inference_steps or 40 + num_steps = req.num_inference_steps or num_inference_steps + + # Respect per-request guidance_scale when explicitly provided. + if req.guidance_scale_provided: + guidance_scale = req.guidance_scale guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0] guidance_high = ( diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index ed0d6e4c6..635da11e6 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -261,11 +261,11 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): prompt: str | None = None, negative_prompt: str | None = None, image: PIL.Image.Image | torch.Tensor | None = None, - height: int | None = None, - width: int | None = None, - num_inference_steps: int | None = None, + height: int = 480, + width: int = 832, + num_inference_steps: int = 40, guidance_scale: float | tuple[float, float] = 5.0, - frame_num: int | None = None, + frame_num: int = 81, output_type: str | None = "np", generator: torch.Generator | None = None, prompt_embeds: torch.Tensor | None = None, @@ -287,10 +287,14 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): if image is None: raise ValueError("Image is required for I2V generation.") - height = req.height or height or 480 - width = req.width or width or 832 - num_frames = req.num_frames if req.num_frames else frame_num or 81 - num_steps = req.num_inference_steps or num_inference_steps or 50 + height = req.height or height + width = req.width or width + num_frames = req.num_frames if req.num_frames else frame_num + num_steps = req.num_inference_steps or num_inference_steps + + # Respect per-request guidance_scale when explicitly provided. + if req.guidance_scale_provided: + guidance_scale = req.guidance_scale # Handle guidance scales guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0] diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py index 6a9a6a6a0..6c1e39242 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py @@ -200,11 +200,11 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): prompt: str | None = None, negative_prompt: str | None = None, image: PIL.Image.Image | torch.Tensor | None = None, - height: int | None = None, - width: int | None = None, - num_inference_steps: int | None = None, + height: int = 704, + width: int = 1280, + num_inference_steps: int = 40, guidance_scale: float = 5.0, - frame_num: int | None = None, + frame_num: int = 81, output_type: str | None = "np", generator: torch.Generator | None = None, prompt_embeds: torch.Tensor | None = None, @@ -223,10 +223,14 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): image = req.pil_image # Default dimensions for TI2V-5B (720P) - height = req.height or height or 704 - width = req.width or width or 1280 - num_frames = req.num_frames if req.num_frames else frame_num or 121 - num_steps = req.num_inference_steps or num_inference_steps or 50 + height = req.height or height + width = req.width or width + num_frames = req.num_frames if req.num_frames else frame_num + num_steps = req.num_inference_steps or num_inference_steps + + # Respect per-request guidance_scale when explicitly provided. + if req.guidance_scale_provided: + guidance_scale = req.guidance_scale self._guidance_scale = guidance_scale diff --git a/vllm_omni/entrypoints/async_omni_diffusion.py b/vllm_omni/entrypoints/async_omni_diffusion.py index 1d47205f1..bfaf7aaae 100644 --- a/vllm_omni/entrypoints/async_omni_diffusion.py +++ b/vllm_omni/entrypoints/async_omni_diffusion.py @@ -138,6 +138,9 @@ class AsyncOmniDiffusion: if key in field_names: init_kwargs[key] = value + if "guidance_scale" in kwargs: + init_kwargs["guidance_scale_provided"] = True + return OmniDiffusionRequest(**init_kwargs) async def generate( From 4f3fd4de3720eafe150482e61c4c57d0c1e4e2b2 Mon Sep 17 00:00:00 2001 From: zhou zhuoxin Date: Mon, 26 Jan 2026 17:23:48 +0800 Subject: [PATCH 17/83] [doc]Add Text-To-Audio Readme documentation (#958) Signed-off-by: zhou zhuoxin --- .../offline_inference/text_to_audio/README.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 examples/offline_inference/text_to_audio/README.md diff --git a/examples/offline_inference/text_to_audio/README.md b/examples/offline_inference/text_to_audio/README.md new file mode 100644 index 000000000..8ec1eafe5 --- /dev/null +++ b/examples/offline_inference/text_to_audio/README.md @@ -0,0 +1,37 @@ +# Text-To-Audio + +The `stabilityai/stable-audio-open-1.0` pipeline generates audio from text prompts. + +## Prerequisites + +If you use a gated model (e.g., `stabilityai/stable-audio-open-1.0`), ensure you have access: + +1. **Accept Model License**: Visit the model page on Hugging Face (e.g., [stabilityai/stable-audio-open-1.0]) and accept the user agreement. +2. **Authenticate**: Log in to Hugging Face locally to access the gated model. + ```bash + huggingface-cli login + ``` + +## Local CLI Usage + +```bash +python text_to_audio.py \ + --model stabilityai/stable-audio-open-1.0 \ + --prompt "The sound of a hammer hitting a wooden surface" \ + --negative_prompt "Low quality" \ + --seed 42 \ + --guidance_scale 7.0 \ + --audio_length 10.0 \ + --num_inference_steps 100 \ + --output stable_audio_output.wav +``` + +Key arguments: + +- `--prompt`: text description (string). +- `--negative_prompt`: negative prompt for classifier-free guidance. +- `--seed`: integer seed for deterministic generation. +- `--guidance_scale`: classifier-free guidance scale. +- `--audio_length`: audio duration in seconds. +- `--num_inference_steps`: diffusion sampling steps.(more steps = higher quality, slower). +- `--output`: path to save the generated WAV file. From 5037af1033ea6af56d5b31d3f3925b64a326274f Mon Sep 17 00:00:00 2001 From: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:22:40 +0800 Subject: [PATCH 18/83] [Feature] Diffusion LoRA Adapter Support (PEFT compatible) for vLLM alignment (#758) Signed-off-by: AndyZhou952 Signed-off-by: Andy Zhou Signed-off-by: dongbo910220 <1275604947@qq.com> Signed-off-by: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com> Signed-off-by: Samit <285365963@qq.com> Co-authored-by: dongbo910220 <1275604947@qq.com> Co-authored-by: Samit <285365963@qq.com> --- .buildkite/pipeline.yml | 17 + .buildkite/scripts/simple_test.sh | 1 + .pre-commit-config.yaml | 5 +- .../offline_inference/lora_inference.md | 107 +++ .../examples/online_serving/lora_inference.md | 69 ++ .../lora_inference/README.md | 98 +++ .../lora_inference/lora_inference.py | 147 ++++ .../online_serving/lora_inference/README.md | 54 ++ .../lora_inference/openai_chat_client.py | 124 ++++ .../lora_inference/run_curl_lora_inference.sh | 61 ++ .../lora_inference/run_server.sh | 20 + tests/diffusion/attention/test_flash_attn.py | 290 -------- tests/diffusion/lora/test_base_linear.py | 171 +++++ tests/diffusion/lora/test_lora_manager.py | 343 ++++++++++ .../offline_inference/test_diffusion_lora.py | 138 ++++ .../test_images_generations_lora.py | 259 +++++++ vllm_omni/config/__init__.py | 2 + vllm_omni/config/lora.py | 8 + vllm_omni/config/model.py | 5 +- vllm_omni/diffusion/data.py | 20 +- vllm_omni/diffusion/lora/__init__.py | 6 + vllm_omni/diffusion/lora/layers/__init__.py | 22 + .../diffusion/lora/layers/base_linear.py | 152 +++++ .../lora/layers/column_parallel_linear.py | 61 ++ .../lora/layers/replicated_linear.py | 20 + .../lora/layers/row_parallel_linear.py | 20 + vllm_omni/diffusion/lora/manager.py | 631 ++++++++++++++++++ vllm_omni/diffusion/lora/utils.py | 87 +++ .../model_loader/diffusers_loader.py | 25 +- .../flux2_klein/flux2_klein_transformer.py | 4 + .../models/glm_image/glm_image_transformer.py | 4 + .../longcat_image_transformer.py | 5 + .../ovis_image/ovis_image_transformer.py | 4 + .../qwen_image/qwen_image_transformer.py | 4 + .../diffusion/models/sd3/sd3_transformer.py | 4 + .../models/wan2_2/wan2_2_transformer.py | 3 + .../models/z_image/z_image_transformer.py | 4 + vllm_omni/diffusion/request.py | 6 + .../diffusion/worker/gpu_diffusion_worker.py | 51 ++ vllm_omni/engine/input_processor.py | 2 +- vllm_omni/entrypoints/async_omni.py | 5 +- vllm_omni/entrypoints/async_omni_diffusion.py | 65 ++ vllm_omni/entrypoints/omni.py | 20 + vllm_omni/entrypoints/openai/api_server.py | 69 ++ .../entrypoints/openai/protocol/images.py | 13 + vllm_omni/entrypoints/openai/serving_chat.py | 31 +- vllm_omni/lora/__init__.py | 0 vllm_omni/lora/request.py | 8 + vllm_omni/lora/utils.py | 22 + 49 files changed, 2977 insertions(+), 310 deletions(-) create mode 100644 docs/user_guide/examples/offline_inference/lora_inference.md create mode 100644 docs/user_guide/examples/online_serving/lora_inference.md create mode 100644 examples/offline_inference/lora_inference/README.md create mode 100644 examples/offline_inference/lora_inference/lora_inference.py create mode 100644 examples/online_serving/lora_inference/README.md create mode 100644 examples/online_serving/lora_inference/openai_chat_client.py create mode 100755 examples/online_serving/lora_inference/run_curl_lora_inference.sh create mode 100755 examples/online_serving/lora_inference/run_server.sh delete mode 100644 tests/diffusion/attention/test_flash_attn.py create mode 100644 tests/diffusion/lora/test_base_linear.py create mode 100644 tests/diffusion/lora/test_lora_manager.py create mode 100644 tests/e2e/offline_inference/test_diffusion_lora.py create mode 100644 tests/e2e/online_serving/test_images_generations_lora.py create mode 100644 vllm_omni/config/lora.py create mode 100644 vllm_omni/diffusion/lora/__init__.py create mode 100644 vllm_omni/diffusion/lora/layers/__init__.py create mode 100644 vllm_omni/diffusion/lora/layers/base_linear.py create mode 100644 vllm_omni/diffusion/lora/layers/column_parallel_linear.py create mode 100644 vllm_omni/diffusion/lora/layers/replicated_linear.py create mode 100644 vllm_omni/diffusion/lora/layers/row_parallel_linear.py create mode 100644 vllm_omni/diffusion/lora/manager.py create mode 100644 vllm_omni/diffusion/lora/utils.py create mode 100644 vllm_omni/lora/__init__.py create mode 100644 vllm_omni/lora/request.py create mode 100644 vllm_omni/lora/utils.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8f942b041..ca7ed7920 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,6 +41,23 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion Images API LoRA E2E" + timeout_in_minutes: 20 + depends_on: image-build + commands: + - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py + agents: + queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion Model CPU offloading Test" timeout_in_minutes: 20 depends_on: image-build diff --git a/.buildkite/scripts/simple_test.sh b/.buildkite/scripts/simple_test.sh index 9ac1cb9e4..e76a5c70d 100755 --- a/.buildkite/scripts/simple_test.sh +++ b/.buildkite/scripts/simple_test.sh @@ -52,6 +52,7 @@ VENV_PYTHON="${VENV_DIR}/bin/python" "${VENV_PYTHON}" -m pytest -v -s tests/entrypoints/ "${VENV_PYTHON}" -m pytest -v -s tests/diffusion/cache/ +"${VENV_PYTHON}" -m pytest -v -s tests/diffusion/lora/ "${VENV_PYTHON}" -m pytest -v -s tests/model_executor/models/qwen2_5_omni/test_audio_length.py "${VENV_PYTHON}" -m pytest -v -s tests/worker/ "${VENV_PYTHON}" -m pytest -v -s tests/distributed/omni_connectors/test_kv_flow.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6e8c3d42d..9e13098f6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,10 @@ repos: # only for staged files - repo: https://github.com/rhysd/actionlint - rev: v1.7.9 + # v1.7.8+ sets `go 1.24.0` in go.mod, which older Go toolchains (and most + # current CI images) cannot parse. Pin to v1.7.7 until actionlint fixes the + # go.mod directive. + rev: v1.7.7 hooks: - id: actionlint files: ^\.github/workflows/.*\.ya?ml$ diff --git a/docs/user_guide/examples/offline_inference/lora_inference.md b/docs/user_guide/examples/offline_inference/lora_inference.md new file mode 100644 index 000000000..dde42655e --- /dev/null +++ b/docs/user_guide/examples/offline_inference/lora_inference.md @@ -0,0 +1,107 @@ +# LoRA-Inference + +Source . + +This contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference. +The example uses the `stabilityai/stable-diffusion-3.5-medium` as the default model, but you can replace it with other models in vLLM-omni. + +## Overview + +Similar to vLLM, vLLM-omni uses a unified LoRA handling mechanism: + +- **Pre-loaded LoRA**: Loaded at initialization via `--lora-path` (pre-loaded into cache) +- **Per-request LoRA**: Loaded on-demand. In the example, the LoRA is loaded via `--lora-request-path` in each request + +Both approaches use the same underlying mechanism - all LoRA adapters are handled uniformly through `set_active_adapter()`. If no LoRA request is provided in a request, all adapters are deactivated. + +## Usage + +### Pre-loaded LoRA (via --lora-path) + +Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests: + +```bash +python -m examples.offline_inference.lora_inference.lora_inference \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora/ \ + --lora-scale 1.0 \ + --num_inference_steps 50 \ + --height 1024 \ + --width 1024 \ + --output output_preloaded.png +``` + +**Note**: When using `--lora-path`, the adapter is loaded at init time with a stable ID derived from the adapter path. This example activates it automatically for the request. + +### Per-request LoRA (via --lora-request-path) + +Load a LoRA adapter on-demand for each request: + +```bash +python -m examples.offline_inference.lora_inference.lora_inference \ + --prompt "A piece of cheesecake" \ + --lora-request-path /path/to/lora/ \ + --lora-scale 1.0 \ + --num_inference_steps 50 \ + --height 1024 \ + --width 1024 \ + --output output_per_request.png +``` + +### No LoRA + +If no LoRA request is provided, we will use the base model without any LoRA adapters: + +```bash +python -m examples.offline_inference.lora_inference.lora_inference \ + --prompt "A piece of cheesecake" \ + --num_inference_steps 50 \ + --height 1024 \ + --width 1024 \ + --output output_no_lora.png +``` + +## Parameters + +### LoRA Parameters + +- `--lora-path`: Path to LoRA adapter folder to pre-load at initialization (loads into cache with a stable ID derived from the path) +- `--lora-request-path`: Path to LoRA adapter folder for per-request loading +- `--lora-request-id`: Integer ID for the LoRA adapter (optional). If not provided and `--lora-request-path` is set, will derive a stable ID from the path. +- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter. + +### Standard Parameters + +- `--prompt`: Text prompt for image generation (required) +- `--seed`: Random seed for reproducibility (default: 42) +- `--height`: Image height in pixels (default: 1024) +- `--width`: Image width in pixels (default: 1024) +- `--num_inference_steps`: Number of denoising steps (default: 50) +- `--output`: Output file path (default: `lora_output.png`) + +## How LoRA Works + +All LoRA adapters are handled uniformly: + +1. **Initialization**: If `--lora-path` is provided, the adapter is loaded into cache with a stable ID derived from the adapter path +2. **Per-request**: If `--lora-request-path` is provided, the adapter is loaded/activated for that request +3. **No LoRA**: If no LoRA request is provided (`req.lora_request` is None), all adapters are deactivated + +The system uses LRU cache management - adapters are cached and evicted when the cache is full (unless pinned). + +## LoRA Adapter Format + +LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` + +## Example materials + +??? abstract "lora_inference.py" + ``````py + --8<-- "examples/offline_inference/lora_inference/lora_inference.py" + `````` diff --git a/docs/user_guide/examples/online_serving/lora_inference.md b/docs/user_guide/examples/online_serving/lora_inference.md new file mode 100644 index 000000000..4c8b215d2 --- /dev/null +++ b/docs/user_guide/examples/online_serving/lora_inference.md @@ -0,0 +1,69 @@ +# LoRA-Inference + +Source . + +This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API. + +> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory). +> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here. + +## Start Server + +```bash +# Pick a diffusion model (examples) +# export MODEL=stabilityai/stable-diffusion-3.5-medium +# export MODEL=Qwen/Qwen-Image + +bash run_server.sh +``` + +## Call API (curl) + +```bash +# Required: local LoRA folder on the server +export LORA_PATH=/path/to/lora_adapter + +# Optional +export SERVER=http://localhost:8091 +export PROMPT="A piece of cheesecake" +export LORA_NAME=my_lora +export LORA_SCALE=1.0 +# Optional: if omitted, the server derives a stable id from LORA_PATH. +# export LORA_INT_ID=123 + +bash run_curl_lora_inference.sh +``` + +## Call API (Python) + +```bash +python openai_chat_client.py \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora_adapter \ + --lora-name my_lora \ + --lora-scale 1.0 \ + --output output.png +``` + +## LoRA Format + +LoRA adapters should be in PEFT format, for example: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` + +??? abstract "openai_chat_client.py" + ``````py + --8<-- "examples/online_serving/lora_inference/openai_chat_client.py" + `````` +??? abstract "run_curl_lora_inference.sh" + ``````py + --8<-- "examples/online_serving/lora_inference/run_curl_lora_inference.sh" + `````` +??? abstract "run_server.sh" + ``````py + --8<-- "examples/online_serving/lora_inference/run_server.sh" + `````` diff --git a/examples/offline_inference/lora_inference/README.md b/examples/offline_inference/lora_inference/README.md new file mode 100644 index 000000000..b0b195f6e --- /dev/null +++ b/examples/offline_inference/lora_inference/README.md @@ -0,0 +1,98 @@ +# LoRA Inference Examples + +This directory contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference. +The example uses the `stabilityai/stable-diffusion-3.5-medium` as the default model, but you can replace it with other models in vLLM-omni. + +## Overview + +Similar to vLLM, vLLM-omni uses a unified LoRA handling mechanism: + +- **Pre-loaded LoRA**: Loaded at initialization via `--lora-path` (pre-loaded into cache) +- **Per-request LoRA**: Loaded on-demand. In the example, the LoRA is loaded via `--lora-request-path` in each request + +Both approaches use the same underlying mechanism - all LoRA adapters are handled uniformly through `set_active_adapter()`. If no LoRA request is provided in a request, all adapters are deactivated. + +## Usage + +### Pre-loaded LoRA (via --lora-path) + +Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests: + +```bash +python -m examples.offline_inference.lora_inference.lora_inference \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora/ \ + --lora-scale 1.0 \ + --num_inference_steps 50 \ + --height 1024 \ + --width 1024 \ + --output output_preloaded.png +``` + +**Note**: When using `--lora-path`, the adapter is loaded at init time with a stable ID derived from the adapter path. This example activates it automatically for the request. + +### Per-request LoRA (via --lora-request-path) + +Load a LoRA adapter on-demand for each request: + +```bash +python -m examples.offline_inference.lora_inference.lora_inference \ + --prompt "A piece of cheesecake" \ + --lora-request-path /path/to/lora/ \ + --lora-scale 1.0 \ + --num_inference_steps 50 \ + --height 1024 \ + --width 1024 \ + --output output_per_request.png +``` + +### No LoRA + +If no LoRA request is provided, we will use the base model without any LoRA adapters: + +```bash +python -m examples.offline_inference.lora_inference.lora_inference \ + --prompt "A piece of cheesecake" \ + --num_inference_steps 50 \ + --height 1024 \ + --width 1024 \ + --output output_no_lora.png +``` + +## Parameters + +### LoRA Parameters + +- `--lora-path`: Path to LoRA adapter folder to pre-load at initialization (loads into cache with a stable ID derived from the path) +- `--lora-request-path`: Path to LoRA adapter folder for per-request loading +- `--lora-request-id`: Integer ID for the LoRA adapter (optional). If not provided and `--lora-request-path` is set, will derive a stable ID from the path. +- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter. + +### Standard Parameters + +- `--prompt`: Text prompt for image generation (required) +- `--seed`: Random seed for reproducibility (default: 42) +- `--height`: Image height in pixels (default: 1024) +- `--width`: Image width in pixels (default: 1024) +- `--num_inference_steps`: Number of denoising steps (default: 50) +- `--output`: Output file path (default: `lora_output.png`) + +## How LoRA Works + +All LoRA adapters are handled uniformly: + +1. **Initialization**: If `--lora-path` is provided, the adapter is loaded into cache with a stable ID derived from the adapter path +2. **Per-request**: If `--lora-request-path` is provided, the adapter is loaded/activated for that request +3. **No LoRA**: If no LoRA request is provided (`req.lora_request` is None), all adapters are deactivated + +The system uses LRU cache management - adapters are cached and evicted when the cache is full (unless pinned). + +## LoRA Adapter Format + +LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` diff --git a/examples/offline_inference/lora_inference/lora_inference.py b/examples/offline_inference/lora_inference/lora_inference.py new file mode 100644 index 000000000..17e9d6196 --- /dev/null +++ b/examples/offline_inference/lora_inference/lora_inference.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import argparse +from pathlib import Path + +from vllm_omni.entrypoints.omni import Omni +from vllm_omni.lora.request import LoRARequest +from vllm_omni.lora.utils import stable_lora_int_id + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate images with LoRA adapters.") + parser.add_argument("--model", default="stabilityai/stable-diffusion-3.5-medium", help="Model name or path.") + parser.add_argument("--prompt", required=True, help="Text prompt for image generation.") + parser.add_argument("--seed", type=int, default=42, help="Random seed for deterministic results.") + parser.add_argument("--height", type=int, default=1024, help="Height of generated image.") + parser.add_argument("--width", type=int, default=1024, help="Width of generated image.") + parser.add_argument( + "--num_inference_steps", + type=int, + default=50, + help="Number of denoising steps for the diffusion sampler.", + ) + parser.add_argument( + "--output", + type=str, + default="lora_output.png", + help="Path to save the generated image (PNG).", + ) + parser.add_argument( + "--lora-path", + type=str, + default=None, + help="Path to LoRA adapter folder to pre-load at initialization (PEFT format). " + "Note: pre-loading populates the cache; you still need to pass a lora_request to activate it.", + ) + parser.add_argument( + "--lora-request-path", + type=str, + default=None, + help="Path to LoRA adapter folder for per-request activation (dynamic LoRA). " + "If --lora-request-id is not provided, a stable ID will be derived from this path.", + ) + parser.add_argument( + "--lora-request-id", + type=int, + default=None, + help="Integer ID for the LoRA adapter (for dynamic LoRA). " + "If not provided and --lora-request-path is set, will derive a stable ID from the path.", + ) + parser.add_argument( + "--lora-scale", + type=float, + default=1.0, + help="Scale factor for LoRA weights (default: 1.0).", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + + model = args.model + + omni_kwargs = {} + + if args.lora_path: + omni_kwargs["lora_path"] = args.lora_path + print(f"Using static LoRA from: {args.lora_path}") + + omni = Omni(model=model, **omni_kwargs) + + lora_request = None + if args.lora_request_path: + if args.lora_request_id is None: + lora_request_id = stable_lora_int_id(args.lora_request_path) + else: + lora_request_id = args.lora_request_id + + lora_name = Path(args.lora_request_path).stem + lora_request = LoRARequest( + lora_name=lora_name, + lora_int_id=lora_request_id, + lora_path=args.lora_request_path, + ) + print(f"Using per-request LoRA: name={lora_name}, id={lora_request_id}, scale={args.lora_scale}") + elif args.lora_path: + # pre-loaded LoRA + lora_request_id = stable_lora_int_id(args.lora_path) + lora_request = LoRARequest( + lora_name="preloaded", + lora_int_id=lora_request_id, + lora_path=args.lora_path, + ) + print(f"Activating pre-loaded LoRA: id={lora_request_id}, scale={args.lora_scale}") + + gen_kwargs = { + "prompt": args.prompt, + "height": args.height, + "width": args.width, + "num_inference_steps": args.num_inference_steps, + } + + if lora_request: + gen_kwargs["lora_request"] = lora_request + gen_kwargs["lora_scale"] = args.lora_scale + + outputs = omni.generate(**gen_kwargs) + + if not outputs or len(outputs) == 0: + raise ValueError("No output generated from omni.generate()") + + if isinstance(outputs, list): + first_output = outputs[0] + else: + first_output = outputs + + images = None + if hasattr(first_output, "images") and first_output.images: + images = first_output.images + elif hasattr(first_output, "request_output") and first_output.request_output: + req_out = first_output.request_output + if isinstance(req_out, list) and len(req_out) > 0: + req_out = req_out[0] + if hasattr(req_out, "images") and req_out.images: + images = req_out.images + + if not images: + raise ValueError("No images found in request_output") + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + suffix = output_path.suffix or ".png" + stem = output_path.stem or "lora_output" + if len(images) <= 1: + images[0].save(output_path) + print(f"Saved generated image to {output_path}") + else: + for idx, img in enumerate(images): + save_path = output_path.parent / f"{stem}_{idx}{suffix}" + img.save(save_path) + print(f"Saved generated image to {save_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/lora_inference/README.md b/examples/online_serving/lora_inference/README.md new file mode 100644 index 000000000..16ce55313 --- /dev/null +++ b/examples/online_serving/lora_inference/README.md @@ -0,0 +1,54 @@ +# Online LoRA Inference (Diffusion) + +This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API. + +> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory). +> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here. + +## Start Server + +```bash +# Pick a diffusion model (examples) +# export MODEL=stabilityai/stable-diffusion-3.5-medium +# export MODEL=Qwen/Qwen-Image + +bash run_server.sh +``` + +## Call API (curl) + +```bash +# Required: local LoRA folder on the server +export LORA_PATH=/path/to/lora_adapter + +# Optional +export SERVER=http://localhost:8091 +export PROMPT="A piece of cheesecake" +export LORA_NAME=my_lora +export LORA_SCALE=1.0 +# Optional: if omitted, the server derives a stable id from LORA_PATH. +# export LORA_INT_ID=123 + +bash run_curl_lora_inference.sh +``` + +## Call API (Python) + +```bash +python openai_chat_client.py \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora_adapter \ + --lora-name my_lora \ + --lora-scale 1.0 \ + --output output.png +``` + +## LoRA Format + +LoRA adapters should be in PEFT format, for example: + +``` +lora_adapter/ +├── adapter_config.json +└── adapter_model.safetensors +``` diff --git a/examples/online_serving/lora_inference/openai_chat_client.py b/examples/online_serving/lora_inference/openai_chat_client.py new file mode 100644 index 000000000..e24d2fdf6 --- /dev/null +++ b/examples/online_serving/lora_inference/openai_chat_client.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +OpenAI-compatible chat client for diffusion LoRA inference. + +Example: + python openai_chat_client.py \ + --server http://localhost:8091 \ + --prompt "A piece of cheesecake" \ + --lora-path /path/to/lora_adapter \ + --lora-name my_lora \ + --lora-scale 1.0 \ + --output output.png +""" + +import argparse +import base64 +from pathlib import Path + +import requests + + +def generate_image( + *, + prompt: str, + server_url: str, + height: int | None, + width: int | None, + num_inference_steps: int | None, + seed: int | None, + lora_name: str | None, + lora_path: str | None, + lora_scale: float | None, + lora_int_id: int | None, +) -> bytes | None: + messages = [{"role": "user", "content": prompt}] + + extra_body: dict = {} + if height is not None: + extra_body["height"] = height + if width is not None: + extra_body["width"] = width + if num_inference_steps is not None: + extra_body["num_inference_steps"] = num_inference_steps + if seed is not None: + extra_body["seed"] = seed + + if lora_path: + lora_body: dict = { + "local_path": lora_path, + "name": lora_name or Path(lora_path).stem, + } + if lora_scale is not None: + lora_body["scale"] = float(lora_scale) + if lora_int_id is not None: + lora_body["int_id"] = int(lora_int_id) + extra_body["lora"] = lora_body + + payload = {"messages": messages} + if extra_body: + payload["extra_body"] = extra_body + + response = requests.post( + f"{server_url}/v1/chat/completions", + headers={"Content-Type": "application/json"}, + json=payload, + timeout=300, + ) + response.raise_for_status() + data = response.json() + + content = data["choices"][0]["message"]["content"] + if isinstance(content, list) and content: + image_url = content[0].get("image_url", {}).get("url", "") + if image_url.startswith("data:image"): + _, b64_data = image_url.split(",", 1) + return base64.b64decode(b64_data) + + raise RuntimeError(f"Unexpected response format: {content!r}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Diffusion LoRA OpenAI chat client") + parser.add_argument("--server", default="http://localhost:8091", help="Server URL") + parser.add_argument("--prompt", default="A piece of cheesecake", help="Text prompt") + parser.add_argument("--output", default="lora_online_output.png", help="Output image path") + + parser.add_argument("--height", type=int, default=1024, help="Image height") + parser.add_argument("--width", type=int, default=1024, help="Image width") + parser.add_argument("--steps", type=int, default=50, help="num_inference_steps") + parser.add_argument("--seed", type=int, default=42, help="Random seed") + + parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)") + parser.add_argument("--lora-name", default=None, help="LoRA name (optional)") + parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale") + parser.add_argument( + "--lora-int-id", + type=int, + default=None, + help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.", + ) + + args = parser.parse_args() + + image_bytes = generate_image( + prompt=args.prompt, + server_url=args.server, + height=args.height, + width=args.width, + num_inference_steps=args.steps, + seed=args.seed, + lora_name=args.lora_name, + lora_path=args.lora_path, + lora_scale=args.lora_scale if args.lora_path else None, + lora_int_id=args.lora_int_id if args.lora_path else None, + ) + + out_path = Path(args.output) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(image_bytes) + print(f"Saved: {out_path} ({len(image_bytes) / 1024:.1f} KiB)") + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/lora_inference/run_curl_lora_inference.sh b/examples/online_serving/lora_inference/run_curl_lora_inference.sh new file mode 100755 index 000000000..14a074fbf --- /dev/null +++ b/examples/online_serving/lora_inference/run_curl_lora_inference.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Online diffusion LoRA inference via OpenAI-compatible chat API. + +SERVER="${SERVER:-http://localhost:8091}" +PROMPT="${PROMPT:-A piece of cheesecake}" + +LORA_PATH="${LORA_PATH:-}" +LORA_NAME="${LORA_NAME:-lora}" +LORA_SCALE="${LORA_SCALE:-1.0}" +LORA_INT_ID="${LORA_INT_ID:-}" + +HEIGHT="${HEIGHT:-1024}" +WIDTH="${WIDTH:-1024}" +NUM_INFERENCE_STEPS="${NUM_INFERENCE_STEPS:-50}" +SEED="${SEED:-42}" + +CURRENT_TIME=$(date +%Y%m%d%H%M%S) +OUTPUT="${OUTPUT:-lora_online_output_${CURRENT_TIME}.png}" + +if [ -z "$LORA_PATH" ]; then + echo "ERROR: LORA_PATH is required (must be a server-local path)." + exit 1 +fi + +echo "Generating image with LoRA..." +echo "Server: $SERVER" +echo "Prompt: $PROMPT" +echo "LoRA: name=$LORA_NAME id=${LORA_INT_ID:-auto} scale=$LORA_SCALE path=$LORA_PATH" +echo "Output: $OUTPUT" + +LORA_INT_ID_FIELD="" +if [ -n "$LORA_INT_ID" ]; then + LORA_INT_ID_FIELD=", \"int_id\": $LORA_INT_ID" +fi + +curl -s "$SERVER/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{ + \"messages\": [ + {\"role\": \"user\", \"content\": \"$PROMPT\"} + ], + \"extra_body\": { + \"height\": $HEIGHT, + \"width\": $WIDTH, + \"num_inference_steps\": $NUM_INFERENCE_STEPS, + \"seed\": $SEED, + \"lora\": { + \"name\": \"$LORA_NAME\", + \"local_path\": \"$LORA_PATH\", + \"scale\": $LORA_SCALE$LORA_INT_ID_FIELD + } + } + }" | jq -r '.choices[0].message.content[0].image_url.url' | sed 's/^data:image[^,]*,\s*//' | base64 -d > "$OUTPUT" + +if [ -f "$OUTPUT" ]; then + echo "Image saved to: $OUTPUT" + echo "Size: $(du -h "$OUTPUT" | cut -f1)" +else + echo "Failed to generate image" + exit 1 +fi diff --git a/examples/online_serving/lora_inference/run_server.sh b/examples/online_serving/lora_inference/run_server.sh new file mode 100755 index 000000000..3233dd773 --- /dev/null +++ b/examples/online_serving/lora_inference/run_server.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Online diffusion serving with vLLM-Omni (OpenAI-compatible API). + +MODEL="${MODEL:-stabilityai/stable-diffusion-3.5-medium}" +PORT="${PORT:-8091}" + +echo "Starting vLLM-Omni diffusion server..." +echo "Model: $MODEL" +echo "Port: $PORT" + +if [ -z "${VLLM_BIN:-}" ]; then + if command -v vllm-omni >/dev/null 2>&1; then + VLLM_BIN="vllm-omni" + else + VLLM_BIN="vllm" + fi +fi + +"$VLLM_BIN" serve "$MODEL" --omni \ + --port "$PORT" diff --git a/tests/diffusion/attention/test_flash_attn.py b/tests/diffusion/attention/test_flash_attn.py deleted file mode 100644 index 3f3862405..000000000 --- a/tests/diffusion/attention/test_flash_attn.py +++ /dev/null @@ -1,290 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -""" -Test script for FlashAttention backend with padding handling. - -This script tests two main scenarios: -1. Case 1: Comparing padded vs unpadded inputs for batch_size=1 -2. Case 2: Comparing FlashAttention and SDPA backends for batch_size=2 with padding -""" - -import pytest -import torch - -from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata -from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl -from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl - - -def create_attention_mask(batch_size: int, seq_len: int, valid_len: int, device: torch.device) -> torch.Tensor: - """ - Create attention mask where first valid_len tokens are valid (1) and rest are padding (0). - - Args: - batch_size: Batch size - seq_len: Total sequence length (including padding) - valid_len: Number of valid (non-padded) tokens - - Returns: - Attention mask of shape (batch_size, seq_len) - """ - mask = torch.zeros(batch_size, seq_len, dtype=torch.bool, device=device) - mask[:, :valid_len] = True - return mask - - -def pad_tensor(tensor: torch.Tensor, target_seq_len: int, pad_value: float = 0.0) -> torch.Tensor: - """ - Pad tensor along sequence dimension (dim=1). - - Args: - tensor: Input tensor of shape (batch_size, seq_len, num_heads, head_dim) - target_seq_len: Target sequence length after padding - pad_value: Value to use for padding - - Returns: - Padded tensor of shape (batch_size, target_seq_len, num_heads, head_dim) - """ - batch_size, seq_len, num_heads, head_dim = tensor.shape - if target_seq_len <= seq_len: - return tensor - - padding = torch.full( - (batch_size, target_seq_len - seq_len, num_heads, head_dim), pad_value, dtype=tensor.dtype, device=tensor.device - ) - return torch.cat([tensor, padding], dim=1) - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="FlashAttention requires CUDA") -def test_padding_equivalence(): - """ - Case 1: Test that padded and unpadded inputs produce similar outputs. - - - Input A: batch_size=1, hidden_states (1, 48), encoder_hidden_states (1, 16) - Concatenated length: 64, NO attention_mask - - Input B: Same data but padded: hidden_states (1, 58), encoder_hidden_states (1, 26) - Concatenated length: 84, WITH attention_mask - - Expected: Output A and Output B should be very close. - """ - device = torch.device("cuda") - dtype = torch.bfloat16 - - # Configuration - batch_size = 1 - hidden_seq_len = 48 - encoder_seq_len = 16 - pad_length = 10 - num_heads = 8 - head_dim = 64 - - # Initialize FlashAttention - fa_impl = FlashAttentionImpl( - num_heads=num_heads, head_size=head_dim, softmax_scale=1.0 / (head_dim**0.5), causal=False - ) - - # Create base tensors with random values (same for both A and B) - torch.manual_seed(42) - hidden_states_base = torch.randn(batch_size, hidden_seq_len, num_heads, head_dim, device=device, dtype=dtype) - encoder_hidden_states_base = torch.randn( - batch_size, encoder_seq_len, num_heads, head_dim, device=device, dtype=dtype - ) - - # ========== Input A: Unpadded, no attention mask ========== - query_a = torch.cat([hidden_states_base, encoder_hidden_states_base], dim=1) - key_a = query_a.clone() - value_a = query_a.clone() - - attn_metadata_a = AttentionMetadata(attn_mask=None) - - output_a = fa_impl.forward(query=query_a, key=key_a, value=value_a, attn_metadata=attn_metadata_a) - - # ========== Input B: Padded with attention mask ========== - hidden_states_padded = pad_tensor(hidden_states_base, hidden_seq_len + pad_length) - encoder_hidden_states_padded = pad_tensor(encoder_hidden_states_base, encoder_seq_len + pad_length) - - query_b = torch.cat([hidden_states_padded, encoder_hidden_states_padded], dim=1) - key_b = query_b.clone() - value_b = query_b.clone() - - # Create attention mask - attn_mask_b = torch.cat( - [ - create_attention_mask(batch_size, hidden_seq_len + pad_length, hidden_seq_len, device), - create_attention_mask(batch_size, encoder_seq_len + pad_length, encoder_seq_len, device), - ], - dim=1, - ) - - attn_metadata_b = AttentionMetadata(attn_mask=attn_mask_b) - - output_b = fa_impl.forward(query=query_b, key=key_b, value=value_b, attn_metadata=attn_metadata_b) - - # Extract non-padded portion from output_b - output_b_unpadded = torch.cat( - [ - output_b[:, :hidden_seq_len, :, :], - output_b[:, hidden_seq_len + pad_length : hidden_seq_len + pad_length + encoder_seq_len, :, :], - ], - dim=1, - ) - - # Compare outputs - max_diff = torch.max(torch.abs(output_a - output_b_unpadded)).item() - mean_diff = torch.mean(torch.abs(output_a - output_b_unpadded)).item() - - print("\n=== Case 1: Padding Equivalence Test ===") - print(f"Output A shape: {output_a.shape}") - print(f"Output B shape: {output_b.shape}") - print(f"Output B unpadded shape: {output_b_unpadded.shape}") - print(f"Max absolute difference: {max_diff:.6f}") - print(f"Mean absolute difference: {mean_diff:.6f}") - - # Assert that outputs are close - # Using higher tolerance for bfloat16 - assert max_diff < 0.1, f"Max difference {max_diff} exceeds threshold 0.1" - assert mean_diff < 0.01, f"Mean difference {mean_diff} exceeds threshold 0.01" - - print("✓ Case 1 PASSED: Padded and unpadded outputs are very close!") - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="FlashAttention requires CUDA") -def test_fa_vs_sdpa(): - """ - Case 2: Compare FlashAttention and SDPA backends with padding. - - - batch_size=2 - - hidden_states: (2, 48) padded to (2, 58) - - encoder_hidden_states: (2, 16) padded to (2, 26) - - Concatenated length: 84 - - Compare FA and SDPA outputs - - Expected: FA and SDPA outputs should be very close. - """ - device = torch.device("cuda") - dtype = torch.bfloat16 - - # Configuration - batch_size = 2 - hidden_seq_len = 48 - encoder_seq_len = 16 - pad_length = 10 - num_heads = 8 - head_dim = 64 - - # Initialize both backends - fa_impl = FlashAttentionImpl( - num_heads=num_heads, head_size=head_dim, softmax_scale=1.0 / (head_dim**0.5), causal=False - ) - - sdpa_impl = SDPAImpl(num_heads=num_heads, head_size=head_dim, softmax_scale=1.0 / (head_dim**0.5), causal=False) - - # Create base tensors - torch.manual_seed(123) - hidden_states_base = torch.randn(batch_size, hidden_seq_len, num_heads, head_dim, device=device, dtype=dtype) - encoder_hidden_states_base = torch.randn( - batch_size, encoder_seq_len, num_heads, head_dim, device=device, dtype=dtype - ) - - # Pad tensors - hidden_states_padded = pad_tensor(hidden_states_base, hidden_seq_len + pad_length) - encoder_hidden_states_padded = pad_tensor(encoder_hidden_states_base, encoder_seq_len + pad_length) - - # Concatenate - query = torch.cat([hidden_states_padded, encoder_hidden_states_padded], dim=1) - key = query.clone() - value = query.clone() - - # Create attention mask - attn_mask = torch.cat( - [ - create_attention_mask(batch_size, hidden_seq_len + pad_length, hidden_seq_len, device), - create_attention_mask(batch_size, encoder_seq_len + pad_length, encoder_seq_len, device), - ], - dim=1, - ) - - attn_metadata = AttentionMetadata(attn_mask=attn_mask) - - # Run FlashAttention - output_fa = fa_impl.forward(query=query.clone(), key=key.clone(), value=value.clone(), attn_metadata=attn_metadata) - - # Run SDPA - # SDPA expects 4D attention mask: (batch_size, 1, seq_len, seq_len) or (batch_size, seq_len) - # For causal=False, we need to convert 2D mask to 4D - if attn_mask is not None: - # Expand mask for SDPA: (batch_size, seq_len) -> (batch_size, 1, 1, seq_len) - attn_mask_4d = attn_mask.unsqueeze(1).unsqueeze(2) - # Convert bool to float: True -> 0.0, False -> -inf - attn_mask_float = torch.zeros_like(attn_mask_4d, dtype=dtype) - attn_mask_float.masked_fill_(~attn_mask_4d, float("-inf")) - attn_metadata_sdpa = AttentionMetadata(attn_mask=attn_mask_float) - else: - attn_metadata_sdpa = AttentionMetadata(attn_mask=None) - - output_sdpa = sdpa_impl.forward( - query=query.clone(), key=key.clone(), value=value.clone(), attn_metadata=attn_metadata_sdpa - ) - - # Compare outputs (only compare valid regions) - output_fa_valid = torch.cat( - [ - output_fa[:, :hidden_seq_len, :, :], - output_fa[:, hidden_seq_len + pad_length : hidden_seq_len + pad_length + encoder_seq_len, :, :], - ], - dim=1, - ) - output_sdpa_valid = torch.cat( - [ - output_sdpa[:, :hidden_seq_len, :, :], - output_sdpa[:, hidden_seq_len + pad_length : hidden_seq_len + pad_length + encoder_seq_len, :, :], - ], - dim=1, - ) - - max_diff = torch.max(torch.abs(output_fa_valid - output_sdpa_valid)).item() - mean_diff = torch.mean(torch.abs(output_fa_valid - output_sdpa_valid)).item() - - print("\n=== Case 2: FA vs SDPA Comparison ===") - print(f"Batch size: {batch_size}") - print(f"FA output shape: {output_fa.shape}") - print(f"SDPA output shape: {output_sdpa.shape}") - print(f"Max absolute difference (valid region): {max_diff:.6f}") - print(f"Mean absolute difference (valid region): {mean_diff:.6f}") - - # Assert that outputs are close - # Using higher tolerance for bfloat16 and different implementations - assert max_diff < 0.01, f"Max difference {max_diff} exceeds threshold 0.01" - assert mean_diff < 0.001, f"Mean difference {mean_diff} exceeds threshold 0.001" - - print("✓ Case 2 PASSED: FA and SDPA outputs are very close!") - - -if __name__ == "__main__": - print("Running FlashAttention Padding Tests...") - print("=" * 60) - - # Try to run CUDA tests - if torch.cuda.is_available(): - try: - print("\n[Running Case 1: Padding Equivalence for FA]") - test_padding_equivalence() - except Exception as e: - print(f"✗ Case 1 failed: {e}") - import traceback - - traceback.print_exc() - - try: - print("\n[Running Case 2: FA vs SDPA]") - test_fa_vs_sdpa() - except Exception as e: - print(f"✗ Case 2 failed: {e}") - import traceback - - traceback.print_exc() - else: - raise RuntimeError("CUDA is not available") - print("\n" + "=" * 60) - print("Test suite completed!") diff --git a/tests/diffusion/lora/test_base_linear.py b/tests/diffusion/lora/test_base_linear.py new file mode 100644 index 000000000..42bdf6526 --- /dev/null +++ b/tests/diffusion/lora/test_base_linear.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from dataclasses import dataclass + +import torch + +from vllm_omni.diffusion.lora.layers.base_linear import DiffusionBaseLinearLayerWithLoRA + + +@dataclass +class _DummyLoRAConfig: + fully_sharded_loras: bool = False + + +class _DummyQuantMethod: + def __init__(self, weight: torch.Tensor): + self._weight = weight + + def apply(self, _base_layer, x: torch.Tensor, bias: torch.Tensor | None): + y = x @ self._weight.t() + if bias is not None: + y = y + bias + return y + + +def test_diffusion_base_linear_apply_multi_slice(): + # Build a fake diffusion LoRA layer with 2 slices and rank=2. + layer = DiffusionBaseLinearLayerWithLoRA.__new__(DiffusionBaseLinearLayerWithLoRA) + layer.tp_size = 1 + layer.lora_config = _DummyLoRAConfig() + + in_dim = 3 + out_slices = (2, 1) + rank = 2 + + # Base weight: identity-ish mapping to make base output easy to reason about. + base_weight = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + layer.base_layer = type("Base", (), {})() + layer.base_layer.quant_method = _DummyQuantMethod(base_weight) + + # Allocate stacked weights: (max_loras=1, 1, rank, in_dim) and (1, 1, out_slice, rank) + a0 = torch.zeros((1, 1, rank, in_dim)) + b0 = torch.zeros((1, 1, out_slices[0], rank)) + a1 = torch.zeros((1, 1, rank, in_dim)) + b1 = torch.zeros((1, 1, out_slices[1], rank)) + + # Slice 0: delta0 = (x @ A0.T) @ B0.T + A0 = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]) # (2, 3) + B0 = torch.tensor([[1.0, 0.0], [0.0, 1.0]]) # (2, 2) + a0[0, 0, :, :] = A0 + b0[0, 0, :, :] = B0 + + # Slice 1: delta1 = (x @ A1.T) @ B1.T + A1 = torch.tensor([[0.0, 0.0, 1.0], [1.0, 0.0, 0.0]]) # (2, 3) + B1 = torch.tensor([[2.0, 0.0]]) # (1, 2) + a1[0, 0, :, :] = A1 + b1[0, 0, :, :] = B1 + + layer.lora_a_stacked = (a0, a1) + layer.lora_b_stacked = (b0, b1) + layer.output_slices = out_slices + + x = torch.tensor([[1.0, 2.0, 3.0]]) + out = layer.apply(x) + + # Base output is identity: [1,2,3] + base_out = x @ base_weight.t() + # delta0: + # (x @ A0.T) = [1,2] + # [1,2] @ B0.T = [1,2] + delta0 = torch.tensor([[1.0, 2.0]]) + # delta1: + # (x @ A1.T) = [3,1] + # [3,1] @ B1.T = [6] + delta1 = torch.tensor([[6.0]]) + expected = torch.cat([base_out[:, :2] + delta0, base_out[:, 2:3] + delta1], dim=-1) + assert torch.allclose(out, expected) + + +def test_diffusion_base_linear_reset_lora_disables_fast_path(monkeypatch): + # Verify that after reset_lora(), apply() skips LoRA matmuls even if the + # LoRA tensors are still allocated and non-empty. + from vllm.lora.layers.base_linear import BaseLinearLayerWithLoRA + + layer = DiffusionBaseLinearLayerWithLoRA.__new__(DiffusionBaseLinearLayerWithLoRA) + layer.tp_size = 1 + layer.lora_config = _DummyLoRAConfig() + + in_dim = 2 + out_dim = 2 + rank = 1 + + base_weight = torch.eye(in_dim) + layer.base_layer = type("Base", (), {})() + layer.base_layer.quant_method = _DummyQuantMethod(base_weight) + + a = torch.ones((1, 1, rank, in_dim)) + b = torch.tensor([[[[1.0], [2.0]]]]) # (1,1,out_dim,rank) + + layer.lora_a_stacked = (a,) + layer.lora_b_stacked = (b,) + layer.output_slices = (out_dim,) + layer._diffusion_lora_active_slices = (True,) + + x = torch.tensor([[1.0, 2.0]]) + out_active = layer.apply(x) + assert torch.allclose(out_active, torch.tensor([[4.0, 8.0]])) + + monkeypatch.setattr(BaseLinearLayerWithLoRA, "reset_lora", lambda self, index: None) + layer.reset_lora(0) + + assert layer._diffusion_lora_active_slices == (False,) + out_inactive = layer.apply(x) + assert torch.allclose(out_inactive, x) + + +def test_diffusion_base_linear_apply_respects_inactive_slices(): + # Build a fake diffusion LoRA layer with 2 slices and rank=2. + layer = DiffusionBaseLinearLayerWithLoRA.__new__(DiffusionBaseLinearLayerWithLoRA) + layer.tp_size = 1 + layer.lora_config = _DummyLoRAConfig() + + in_dim = 3 + out_slices = (2, 1) + rank = 2 + + base_weight = torch.tensor( + [ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + ] + ) + layer.base_layer = type("Base", (), {})() + layer.base_layer.quant_method = _DummyQuantMethod(base_weight) + + a0 = torch.zeros((1, 1, rank, in_dim)) + b0 = torch.zeros((1, 1, out_slices[0], rank)) + a1 = torch.zeros((1, 1, rank, in_dim)) + b1 = torch.zeros((1, 1, out_slices[1], rank)) + + A0 = torch.tensor([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]]) # (2, 3) + B0 = torch.tensor([[1.0, 0.0], [0.0, 1.0]]) # (2, 2) + a0[0, 0, :, :] = A0 + b0[0, 0, :, :] = B0 + + A1 = torch.tensor([[0.0, 0.0, 1.0], [1.0, 0.0, 0.0]]) # (2, 3) + B1 = torch.tensor([[2.0, 0.0]]) # (1, 2) + a1[0, 0, :, :] = A1 + b1[0, 0, :, :] = B1 + + layer.lora_a_stacked = (a0, a1) + layer.lora_b_stacked = (b0, b1) + layer.output_slices = out_slices + layer._diffusion_lora_active_slices = (True, False) + + x = torch.tensor([[1.0, 2.0, 3.0]]) + out = layer.apply(x) + + # Only the first slice should be adapted. + expected = torch.tensor([[2.0, 4.0, 3.0]]) + assert torch.allclose(out, expected) diff --git a/tests/diffusion/lora/test_lora_manager.py b/tests/diffusion/lora/test_lora_manager.py new file mode 100644 index 000000000..84fafe3bc --- /dev/null +++ b/tests/diffusion/lora/test_lora_manager.py @@ -0,0 +1,343 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import torch +from vllm.lora.lora_weights import LoRALayerWeights +from vllm.lora.utils import get_supported_lora_modules +from vllm.model_executor.layers.linear import LinearBase + +from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager +from vllm_omni.lora.request import LoRARequest + + +class _DummyLoRALayer: + def __init__(self, n_slices: int, output_slices: tuple[int, ...]): + self.n_slices = n_slices + self.output_slices = output_slices + self.set_calls: list[ + tuple[list[torch.Tensor | None] | torch.Tensor, list[torch.Tensor | None] | torch.Tensor] + ] = [] + self.reset_calls: int = 0 + + def set_lora(self, index: int, lora_a, lora_b): + assert index == 0 + self.set_calls.append((lora_a, lora_b)) + + def reset_lora(self, index: int): + assert index == 0 + self.reset_calls += 1 + + +class _FakeLinearBase(LinearBase): + def __init__(self): + torch.nn.Module.__init__(self) + + +def test_lora_manager_supported_modules_are_stable_with_wrapped_layers(monkeypatch): + # Simulate a pipeline that already contains LoRA wrappers where the original + # LinearBase is nested under ".base_layer". + import vllm_omni.diffusion.lora.manager as manager_mod + + class _DummyBaseLayerWithLoRA(torch.nn.Module): + def __init__(self, base_layer: torch.nn.Module): + super().__init__() + self.base_layer = base_layer + + monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA) + + pipeline = torch.nn.Module() + pipeline.transformer = torch.nn.Module() + pipeline.transformer.foo = _DummyBaseLayerWithLoRA(_FakeLinearBase()) + + # vLLM helper would see only the nested LinearBase and yield "base_layer". + assert get_supported_lora_modules(pipeline) == ["base_layer"] + + manager = DiffusionLoRAManager( + pipeline=pipeline, + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=1, + ) + + assert "foo" in manager._supported_lora_modules + assert "base_layer" not in manager._supported_lora_modules + + +def test_lora_manager_replace_layers_does_not_rewrap_base_layer(monkeypatch): + import vllm_omni.diffusion.lora.manager as manager_mod + + class _DummyBaseLayerWithLoRA(torch.nn.Module): + def __init__(self, base_layer: torch.nn.Module): + super().__init__() + self.base_layer = base_layer + + monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA) + + def _fake_from_layer_diffusion(*, layer: torch.nn.Module, **_kwargs): + if isinstance(layer, _FakeLinearBase): + return _DummyBaseLayerWithLoRA(layer) + return layer + + replace_calls: list[str] = [] + + def _fake_replace_submodule(root: torch.nn.Module, module_name: str, submodule: torch.nn.Module): + replace_calls.append(module_name) + setattr(root, module_name, submodule) + + monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion) + monkeypatch.setattr(manager_mod, "replace_submodule", _fake_replace_submodule) + + pipeline = torch.nn.Module() + pipeline.transformer = torch.nn.Module() + pipeline.transformer.foo = _FakeLinearBase() + + manager = DiffusionLoRAManager( + pipeline=pipeline, + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=1, + ) + + peft_helper = type("_PH", (), {"r": 1})() + + manager._replace_layers_with_lora(peft_helper) + manager._replace_layers_with_lora(peft_helper) + + # Only the top-level layer should have been replaced; nested ".base_layer" + # must be skipped to avoid nesting LoRA wrappers. + assert replace_calls == ["foo"] + + +def test_lora_manager_replaces_packed_layer_when_targeting_sublayers(monkeypatch): + import vllm_omni.diffusion.lora.manager as manager_mod + + class _DummyBaseLayerWithLoRA(torch.nn.Module): + def __init__(self, base_layer: torch.nn.Module): + super().__init__() + self.base_layer = base_layer + + monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA) + + def _fake_from_layer_diffusion(*, layer: torch.nn.Module, **_kwargs): + return _DummyBaseLayerWithLoRA(layer) + + replace_calls: list[str] = [] + + def _fake_replace_submodule(root: torch.nn.Module, module_name: str, submodule: torch.nn.Module): + replace_calls.append(module_name) + setattr(root, module_name, submodule) + + monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion) + monkeypatch.setattr(manager_mod, "replace_submodule", _fake_replace_submodule) + + pipeline = torch.nn.Module() + pipeline.packed_modules_mapping = {"to_qkv": ["to_q", "to_k", "to_v"]} + pipeline.transformer = torch.nn.Module() + pipeline.transformer.to_qkv = _FakeLinearBase() + + manager = DiffusionLoRAManager( + pipeline=pipeline, + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=1, + ) + + # Treat the dummy layer as a packed 3-slice projection so the manager uses + # `packed_modules_mapping` to decide replacement based on target_modules. + monkeypatch.setattr(manager, "_get_packed_modules_list", lambda _module: ["q", "k", "v"]) + + peft_helper = type("_PH", (), {"r": 1, "target_modules": ["to_q"]})() + manager._replace_layers_with_lora(peft_helper) + + assert replace_calls == ["to_qkv"] + + +def test_lora_manager_activates_fused_lora_on_packed_layer(): + manager = DiffusionLoRAManager( + pipeline=torch.nn.Module(), + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=1, + ) + + packed_layer = _DummyLoRALayer(n_slices=3, output_slices=(2, 1, 1)) + manager._lora_modules = {"transformer.blocks.0.attn.to_qkv": packed_layer} + + rank = 2 + A = torch.ones((rank, 4)) + B = torch.arange(0, sum(packed_layer.output_slices) * rank, dtype=torch.bfloat16).view(-1, rank) + lora = LoRALayerWeights( + module_name="transformer.blocks.0.attn.to_qkv", + rank=rank, + lora_alpha=rank, + lora_a=A, + lora_b=B, + ) + manager._registered_adapters = { + 7: type( + "LM", + (), + { + "id": 7, + "loras": {"transformer.blocks.0.attn.to_qkv": lora}, + "get_lora": lambda self, k: self.loras.get(k), + }, + )() + } + manager._adapter_scales = {7: 0.5} + + manager._activate_adapter(7) + + assert packed_layer.reset_calls == 0 + assert len(packed_layer.set_calls) == 1 + lora_a_list, lora_b_list = packed_layer.set_calls[0] + assert isinstance(lora_a_list, list) + assert isinstance(lora_b_list, list) + assert len(lora_a_list) == 3 + assert len(lora_b_list) == 3 + assert all(torch.allclose(a, A) for a in lora_a_list) + # B should be split into 3 slices and scaled. + b0, b1, b2 = lora_b_list + assert b0.shape[0] == 2 and b1.shape[0] == 1 and b2.shape[0] == 1 + assert torch.allclose(torch.cat([b0, b1, b2], dim=0), B * 0.5) + + +def test_lora_manager_activates_packed_lora_from_sublayers(): + pipeline = torch.nn.Module() + pipeline.packed_modules_mapping = {"to_qkv": ["to_q", "to_k", "to_v"]} + manager = DiffusionLoRAManager( + pipeline=pipeline, + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=1, + ) + + packed_layer = _DummyLoRALayer(n_slices=3, output_slices=(2, 1, 1)) + manager._lora_modules = {"transformer.blocks.0.attn.to_qkv": packed_layer} + + rank = 2 + loras: dict[str, LoRALayerWeights] = {} + for name, out_dim in zip(["to_q", "to_k", "to_v"], [2, 1, 1]): + loras[f"transformer.blocks.0.attn.{name}"] = LoRALayerWeights( + module_name=f"transformer.blocks.0.attn.{name}", + rank=rank, + lora_alpha=rank, + lora_a=torch.ones((rank, 4)) * (1 if name == "to_q" else 2), + lora_b=torch.ones((out_dim, rank)) * (3 if name == "to_q" else 4), + ) + + manager._registered_adapters = { + 1: type("LM", (), {"id": 1, "loras": loras, "get_lora": lambda self, k: self.loras.get(k)})() + } + manager._adapter_scales = {1: 2.0} + + manager._activate_adapter(1) + + assert packed_layer.reset_calls == 0 + assert len(packed_layer.set_calls) == 1 + lora_a_list, lora_b_list = packed_layer.set_calls[0] + assert isinstance(lora_a_list, list) + assert isinstance(lora_b_list, list) + assert len(lora_a_list) == 3 + assert len(lora_b_list) == 3 + # Scale should apply to B only. + assert torch.allclose(lora_b_list[0], torch.ones((2, rank)) * 3 * 2.0) + assert torch.allclose(lora_b_list[1], torch.ones((1, rank)) * 4 * 2.0) + assert torch.allclose(lora_b_list[2], torch.ones((1, rank)) * 4 * 2.0) + + +def _dummy_lora_request(adapter_id: int) -> LoRARequest: + return LoRARequest( + lora_name=f"adapter_{adapter_id}", + lora_int_id=adapter_id, + lora_path=f"/tmp/adapter_{adapter_id}", + ) + + +def test_lora_manager_evicts_lru_adapter_when_cache_full(monkeypatch): + manager = DiffusionLoRAManager( + pipeline=torch.nn.Module(), + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=2, + ) + + def _fake_load(_req: LoRARequest): + lora_model = type("LM", (), {"id": _req.lora_int_id})() + peft_helper = type("PH", (), {})() + return lora_model, peft_helper + + monkeypatch.setattr(manager, "_load_adapter", _fake_load) + monkeypatch.setattr(manager, "_replace_layers_with_lora", lambda _peft: None) + monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id: None) + + req1 = _dummy_lora_request(1) + req2 = _dummy_lora_request(2) + req3 = _dummy_lora_request(3) + + manager.set_active_adapter(req1, lora_scale=1.0) + manager.set_active_adapter(req2, lora_scale=1.0) + + # Touch adapter 1 so adapter 2 becomes LRU. + manager.set_active_adapter(req1, lora_scale=1.0) + + manager.set_active_adapter(req3, lora_scale=1.0) + + assert set(manager.list_adapters()) == {1, 3} + + +def test_lora_manager_does_not_evict_pinned_adapter(monkeypatch): + manager = DiffusionLoRAManager( + pipeline=torch.nn.Module(), + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=2, + ) + + def _fake_load(_req: LoRARequest): + lora_model = type("LM", (), {"id": _req.lora_int_id})() + peft_helper = type("PH", (), {})() + return lora_model, peft_helper + + monkeypatch.setattr(manager, "_load_adapter", _fake_load) + monkeypatch.setattr(manager, "_replace_layers_with_lora", lambda _peft: None) + monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id: None) + + manager.set_active_adapter(_dummy_lora_request(1), lora_scale=1.0) + assert manager.pin_adapter(1) + + manager.set_active_adapter(_dummy_lora_request(2), lora_scale=1.0) + manager.set_active_adapter(_dummy_lora_request(3), lora_scale=1.0) + + assert set(manager.list_adapters()) == {1, 3} + + +def test_lora_manager_warns_when_all_adapters_pinned(monkeypatch): + manager = DiffusionLoRAManager( + pipeline=torch.nn.Module(), + device=torch.device("cpu"), + dtype=torch.bfloat16, + max_cached_adapters=2, + ) + + def _fake_load(_req: LoRARequest): + lora_model = type("LM", (), {"id": _req.lora_int_id})() + peft_helper = type("PH", (), {})() + return lora_model, peft_helper + + monkeypatch.setattr(manager, "_load_adapter", _fake_load) + monkeypatch.setattr(manager, "_replace_layers_with_lora", lambda _peft: None) + monkeypatch.setattr(manager, "_activate_adapter", lambda _adapter_id: None) + + manager.set_active_adapter(_dummy_lora_request(1), lora_scale=1.0) + manager.set_active_adapter(_dummy_lora_request(2), lora_scale=1.0) + + assert manager.pin_adapter(1) + assert manager.pin_adapter(2) + + manager.max_cached_adapters = 1 + manager._evict_if_needed() + + assert set(manager.list_adapters()) == {1, 2} diff --git a/tests/e2e/offline_inference/test_diffusion_lora.py b/tests/e2e/offline_inference/test_diffusion_lora.py new file mode 100644 index 000000000..1761f253c --- /dev/null +++ b/tests/e2e/offline_inference/test_diffusion_lora.py @@ -0,0 +1,138 @@ +import json +import os +import sys +from pathlib import Path + +import pytest +import torch +from safetensors.torch import save_file + +from vllm_omni.outputs import OmniRequestOutput +from vllm_omni.utils.platform_utils import is_npu + +# ruff: noqa: E402 +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from vllm_omni import Omni + +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" + + +# This test is specific to Z-Image LoRA behavior. Keep it focused on a single +# model to reduce runtime and avoid extra downloads. +models = ["Tongyi-MAI/Z-Image-Turbo"] + +# NPU still can't run Tongyi-MAI/Z-Image-Turbo properly. +if is_npu(): + pytest.skip("Tongyi-MAI/Z-Image-Turbo is not supported on NPU yet.", allow_module_level=True) + + +@pytest.mark.parametrize("model_name", models) +def test_diffusion_model(model_name: str, tmp_path: Path): + def _extract_images(outputs: list[OmniRequestOutput]): + if not outputs: + raise ValueError("Empty outputs from Omni.generate()") + first_output = outputs[0] + assert first_output.final_output_type == "image" + if not hasattr(first_output, "request_output") or not first_output.request_output: + raise ValueError("No request_output found in OmniRequestOutput") + + req_out = first_output.request_output[0] + if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"): + raise ValueError("Invalid request_output structure or missing 'images' key") + return req_out.images + + def _write_zimage_lora(adapter_dir: Path) -> str: + adapter_dir.mkdir(parents=True, exist_ok=True) + + # Z-Image transformer uses dim=3840 by default (see ZImageTransformer2DModel). + dim = 3840 + module_name = "transformer.layers.0.attention.to_qkv" + rank = 1 + lora_a = torch.zeros((rank, dim), dtype=torch.float32) + lora_a[0, 0] = 1.0 + + # QKVParallelLinear packs (Q, K, V). With tp=1 and n_kv_heads==n_heads in Z-Image, + # each slice is `dim`, so total out dim is `3 * dim`. + lora_b = torch.zeros((3 * dim, rank), dtype=torch.float32) + # Apply a visible delta to the Q slice only to keep the perturbation bounded. + lora_b[:dim, 0] = 0.1 + + save_file( + { + f"base_model.model.{module_name}.lora_A.weight": lora_a, + f"base_model.model.{module_name}.lora_B.weight": lora_b, + }, + str(adapter_dir / "adapter_model.safetensors"), + ) + (adapter_dir / "adapter_config.json").write_text( + json.dumps( + { + "r": rank, + "lora_alpha": rank, + "target_modules": [module_name], + } + ), + encoding="utf-8", + ) + return str(adapter_dir) + + m = Omni(model=model_name) + try: + # high resolution may cause OOM on L4 + height = 256 + width = 256 + prompt = "a photo of a cat sitting on a laptop keyboard" + + outputs = m.generate( + prompt, + height=height, + width=width, + num_inference_steps=2, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, + ) + images = _extract_images(outputs) + + assert len(images) == 1 + # check image size + assert images[0].width == width + assert images[0].height == height + + # Real LoRA E2E: generate again with a real on-disk PEFT adapter and + # verify that output changes. + if model_name == "Tongyi-MAI/Z-Image-Turbo": + from vllm_omni.lora.request import LoRARequest + from vllm_omni.lora.utils import stable_lora_int_id + + lora_dir = _write_zimage_lora(tmp_path / "zimage_lora") + lora_request = LoRARequest( + lora_name="test", + lora_int_id=stable_lora_int_id(lora_dir), + lora_path=lora_dir, + ) + outputs_lora = m.generate( + prompt, + height=height, + width=width, + num_inference_steps=2, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, + lora_request=lora_request, + lora_scale=2.0, + ) + images_lora = _extract_images(outputs_lora) + assert len(images_lora) == 1 + assert images_lora[0].width == width + assert images_lora[0].height == height + + import numpy as np + + diff = np.abs(np.array(images[0], dtype=np.int16) - np.array(images_lora[0], dtype=np.int16)).mean() + assert diff > 0.0 + finally: + m.close() diff --git a/tests/e2e/online_serving/test_images_generations_lora.py b/tests/e2e/online_serving/test_images_generations_lora.py new file mode 100644 index 000000000..38d3c9b89 --- /dev/null +++ b/tests/e2e/online_serving/test_images_generations_lora.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +E2E online serving test for /v1/images/generations with per-request LoRA. + +This validates: +- The API server accepts a per-request `lora` object in the Images API payload. +- LoRA can be switched per request (adapter A -> adapter B -> no LoRA). +- Output correctness is asserted using a small image slice with tolerance. +""" + +import base64 +import json +import os +import signal +import subprocess +import sys +import time +from io import BytesIO +from pathlib import Path + +import numpy as np +import pytest +import requests +import torch +from PIL import Image +from safetensors.torch import save_file +from vllm.utils.network_utils import get_open_port + +from vllm_omni.utils.platform_utils import is_npu + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = "Tongyi-MAI/Z-Image-Turbo" + + +PROMPT = "a photo of a cat sitting on a laptop keyboard" +SIZE = "256x256" +SEED = 42 + + +class OmniServer: + """Omniserver for vLLM-Omni tests.""" + + def __init__( + self, + model: str, + serve_args: list[str], + *, + env_dict: dict[str, str] | None = None, + ) -> None: + self.model = model + self.serve_args = serve_args + self.env_dict = env_dict + self.proc: subprocess.Popen | None = None + self.host = "127.0.0.1" + self.port = get_open_port() + + def _start_server(self) -> None: + env = os.environ.copy() + env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + if self.env_dict is not None: + env.update(self.env_dict) + + cmd = [ + sys.executable, + "-m", + "vllm_omni.entrypoints.cli.main", + "serve", + self.model, + "--omni", + "--host", + self.host, + "--port", + str(self.port), + ] + self.serve_args + + print(f"Launching OmniServer with: {' '.join(cmd)}") + self.proc = subprocess.Popen( + cmd, + env=env, + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), # vllm-omni root + start_new_session=True, + ) + + # Wait for server to be ready. + max_wait = 600 + url = f"http://{self.host}:{self.port}/v1/models" + start_time = time.time() + while time.time() - start_time < max_wait: + try: + resp = requests.get(url, headers={"Authorization": "Bearer EMPTY"}, timeout=10) + if resp.status_code == 200: + print(f"Server ready on {self.host}:{self.port}") + return + except Exception: + pass + time.sleep(2) + + raise RuntimeError(f"Server failed to become ready within {max_wait} seconds") + + def __enter__(self): + self._start_server() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.proc is None: + return + try: + os.killpg(self.proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + self.proc.wait(timeout=30) + except subprocess.TimeoutExpired: + try: + os.killpg(self.proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + self.proc.wait() + + +@pytest.fixture(scope="module") +def omni_server(): + if is_npu(): + pytest.skip("Tongyi-MAI/Z-Image-Turbo is not supported on NPU yet.") + with OmniServer(MODEL, ["--num-gpus", "1"]) as server: + yield server + + +def _write_zimage_lora(adapter_dir: Path, *, q_scale: float = 0.0, k_scale: float = 0.0, v_scale: float = 0.0): + adapter_dir.mkdir(parents=True, exist_ok=True) + + # Z-Image transformer uses dim=3840 by default. + dim = 3840 + module_name = "transformer.layers.0.attention.to_qkv" + rank = 1 + + lora_a = torch.zeros((rank, dim), dtype=torch.float32) + lora_a[0, 0] = 1.0 + + # QKVParallelLinear packs (Q, K, V) => out dim is 3 * dim (tp=1). + lora_b = torch.zeros((3 * dim, rank), dtype=torch.float32) + if q_scale: + lora_b[:dim, 0] = q_scale + if k_scale: + lora_b[dim : 2 * dim, 0] = k_scale + if v_scale: + lora_b[2 * dim :, 0] = v_scale + + save_file( + { + f"base_model.model.{module_name}.lora_A.weight": lora_a, + f"base_model.model.{module_name}.lora_B.weight": lora_b, + }, + str(adapter_dir / "adapter_model.safetensors"), + ) + (adapter_dir / "adapter_config.json").write_text( + json.dumps( + { + "r": rank, + "lora_alpha": rank, + "target_modules": [module_name], + } + ), + encoding="utf-8", + ) + + +def _post_images(server: OmniServer, payload: dict) -> Image.Image: + url = f"http://{server.host}:{server.port}/v1/images/generations" + resp = requests.post(url, json=payload, headers={"Authorization": "Bearer EMPTY"}, timeout=900) + resp.raise_for_status() + data = resp.json() + b64 = data["data"][0]["b64_json"] + img_bytes = base64.b64decode(b64) + img = Image.open(BytesIO(img_bytes)) + img.load() + return img.convert("RGB") + + +def _image_blue_tail_slice(img: Image.Image) -> np.ndarray: + arr = np.asarray(img, dtype=np.uint8) + assert arr.ndim == 3 and arr.shape[-1] == 3 + tail = arr[-3:, -3:, -1].astype(np.float32) + assert tail.shape == (3, 3) + return tail + + +def _assert_slice_close(actual: np.ndarray, expected: np.ndarray, *, label: str) -> None: + assert actual.shape == (3, 3) + assert expected.shape == (3, 3) + diff = np.abs(actual - expected) + max_diff = float(diff.max()) + mean_diff = float(diff.mean()) + # NOTE: Different attention backends / torch.compile can introduce small + # floating-point drift that shows up as a few LSBs in uint8 pixels. Keep + # the reset check tolerant but bounded to avoid flaky CI. + assert max_diff <= 5.0 and mean_diff <= 3.0, ( + f"{label} slice mismatch (max={max_diff:.1f}, mean={mean_diff:.1f}): {actual.tolist()}" + ) + + +def _assert_slice_diff(actual: np.ndarray, baseline: np.ndarray, *, label: str) -> None: + assert actual.shape == (3, 3) + assert baseline.shape == (3, 3) + diff = np.abs(actual - baseline).mean() + assert diff > 0.1, f"{label} slice diff too small: {diff} ({actual.tolist()} vs {baseline.tolist()})" + + +def _basic_payload() -> dict: + return { + "prompt": PROMPT, + "n": 1, + "size": SIZE, + "num_inference_steps": 2, + "guidance_scale": 0.0, + "seed": SEED, + } + + +def test_images_generations_per_request_lora_switching(omni_server: OmniServer, tmp_path: Path) -> None: + # Base generation. + base_img = _post_images(omni_server, _basic_payload()) + base_slice = _image_blue_tail_slice(base_img) + + # Adapter A: apply delta to Q slice only. + lora_a_dir = tmp_path / "zimage_lora_a" + _write_zimage_lora(lora_a_dir, q_scale=0.1) + payload_a = _basic_payload() + payload_a["lora"] = {"name": "a", "path": str(lora_a_dir), "scale": 2.0} + img_a = _post_images(omni_server, payload_a) + a_slice = _image_blue_tail_slice(img_a) + _assert_slice_diff(a_slice, base_slice, label="lora_a_vs_base") + a_vs_base = float(np.abs(a_slice - base_slice).mean()) + + # Adapter B: apply delta to K slice only (should differ from adapter A). + lora_b_dir = tmp_path / "zimage_lora_b" + _write_zimage_lora(lora_b_dir, k_scale=0.1) + payload_b = _basic_payload() + payload_b["lora"] = {"name": "b", "path": str(lora_b_dir), "scale": 2.0} + img_b = _post_images(omni_server, payload_b) + b_slice = _image_blue_tail_slice(img_b) + _assert_slice_diff(b_slice, base_slice, label="lora_b_vs_base") + _assert_slice_diff(b_slice, a_slice, label="lora_b_vs_lora_a") + b_vs_base = float(np.abs(b_slice - base_slice).mean()) + b_vs_a = float(np.abs(b_slice - a_slice).mean()) + + # Ensure switching back to no-LoRA restores the base output. + base_img_2 = _post_images(omni_server, _basic_payload()) + base_slice_2 = _image_blue_tail_slice(base_img_2) + _assert_slice_close(base_slice_2, base_slice, label="base_after_reset") + base_reset = float(np.abs(base_slice_2 - base_slice).mean()) + + # Ensure LoRA effects are clearly above the baseline drift. + min_delta = base_reset + 0.5 + assert a_vs_base > min_delta, f"lora_a_vs_base drift too small: {a_vs_base} <= {min_delta}" + assert b_vs_base > min_delta, f"lora_b_vs_base drift too small: {b_vs_base} <= {min_delta}" + assert b_vs_a > min_delta, f"lora_b_vs_lora_a drift too small: {b_vs_a} <= {min_delta}" diff --git a/vllm_omni/config/__init__.py b/vllm_omni/config/__init__.py index cc7e8132e..e2db6f427 100644 --- a/vllm_omni/config/__init__.py +++ b/vllm_omni/config/__init__.py @@ -2,8 +2,10 @@ Configuration module for vLLM-Omni. """ +from vllm_omni.config.lora import LoRAConfig from vllm_omni.config.model import OmniModelConfig __all__ = [ "OmniModelConfig", + "LoRAConfig", ] diff --git a/vllm_omni/config/lora.py b/vllm_omni/config/lora.py new file mode 100644 index 000000000..00aba2e16 --- /dev/null +++ b/vllm_omni/config/lora.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# for now, it suffices to use vLLM's implementation directly +# as this is a user-facing variable, defined here to so that user can directly import LoRAConfig from vllm_omni +from vllm.config.lora import LoRAConfig + +__all__ = ["LoRAConfig"] diff --git a/vllm_omni/config/model.py b/vllm_omni/config/model.py index 4fe136288..751a487c3 100644 --- a/vllm_omni/config/model.py +++ b/vllm_omni/config/model.py @@ -21,10 +21,7 @@ from vllm.transformers_utils.config import ( get_hf_text_config, get_pooling_config, ) -from vllm.transformers_utils.gguf_utils import ( - is_gguf, - maybe_patch_hf_config_from_gguf, -) +from vllm.transformers_utils.gguf_utils import is_gguf, maybe_patch_hf_config_from_gguf from vllm.transformers_utils.utils import maybe_model_redirect from vllm.v1.attention.backends.registry import AttentionBackendEnum diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index f85903b9b..1a98462aa 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -276,12 +276,9 @@ class OmniDiffusionConfig: # pipeline_config: PipelineConfig = field(default_factory=PipelineConfig, repr=False) # LoRA parameters - # (Wenxuan) prefer to keep it here instead of in pipeline config to not make it complicated. lora_path: str | None = None - lora_nickname: str = "default" # for swapping adapters in the pipeline - # can restrict layers to adapt, e.g. ["q_proj"] - # Will adapt only q, k, v, o by default. - lora_target_modules: list[str] | None = None + lora_scale: float = 1.0 + max_cpu_loras: int | None = None output_type: str = "pil" @@ -446,11 +443,24 @@ class OmniDiffusionConfig: # If it's neither dict nor DiffusionCacheConfig, convert to empty config self.cache_config = DiffusionCacheConfig() + if self.max_cpu_loras is None: + self.max_cpu_loras = 1 + elif self.max_cpu_loras < 1: + raise ValueError("max_cpu_loras must be >= 1 for diffusion LoRA") + def update_multimodal_support(self) -> None: self.supports_multimodal_inputs = self.model_class_name in {"QwenImageEditPlusPipeline"} @classmethod def from_kwargs(cls, **kwargs: Any) -> "OmniDiffusionConfig": + # Backwards-compatibility: older callers may use a diffusion-specific + # "static_lora_scale" kwarg. Normalize it to the canonical "lora_scale" + # before constructing the dataclass to avoid TypeError on unknown fields. + if "static_lora_scale" in kwargs: + if "lora_scale" not in kwargs: + kwargs["lora_scale"] = kwargs["static_lora_scale"] + kwargs.pop("static_lora_scale", None) + # Check environment variable as fallback for cache_backend # Support both old DIFFUSION_CACHE_ADAPTER and new DIFFUSION_CACHE_BACKEND for backwards compatibility if "cache_backend" not in kwargs: diff --git a/vllm_omni/diffusion/lora/__init__.py b/vllm_omni/diffusion/lora/__init__.py new file mode 100644 index 000000000..353a2c6be --- /dev/null +++ b/vllm_omni/diffusion/lora/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager + +__all__ = ["DiffusionLoRAManager"] diff --git a/vllm_omni/diffusion/lora/layers/__init__.py b/vllm_omni/diffusion/lora/layers/__init__.py new file mode 100644 index 000000000..ab501f105 --- /dev/null +++ b/vllm_omni/diffusion/lora/layers/__init__.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from .base_linear import DiffusionBaseLinearLayerWithLoRA +from .column_parallel_linear import ( + DiffusionColumnParallelLinearWithLoRA, + DiffusionMergedColumnParallelLinearWithLoRA, + DiffusionMergedQKVParallelLinearWithLoRA, + DiffusionQKVParallelLinearWithLoRA, +) +from .replicated_linear import DiffusionReplicatedLinearWithLoRA +from .row_parallel_linear import DiffusionRowParallelLinearWithLoRA + +__all__ = [ + "DiffusionBaseLinearLayerWithLoRA", + "DiffusionReplicatedLinearWithLoRA", + "DiffusionColumnParallelLinearWithLoRA", + "DiffusionMergedColumnParallelLinearWithLoRA", + "DiffusionRowParallelLinearWithLoRA", + "DiffusionQKVParallelLinearWithLoRA", + "DiffusionMergedQKVParallelLinearWithLoRA", +] diff --git a/vllm_omni/diffusion/lora/layers/base_linear.py b/vllm_omni/diffusion/lora/layers/base_linear.py new file mode 100644 index 000000000..fe32868d0 --- /dev/null +++ b/vllm_omni/diffusion/lora/layers/base_linear.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import torch +from vllm.lora.layers.base_linear import BaseLinearLayerWithLoRA + + +class DiffusionBaseLinearLayerWithLoRA(BaseLinearLayerWithLoRA): + """ + Diffusion-specific base that overrides apply() to use direct torch matmul + instead of punica_wrapper. + + punica_wrapper is used to hold multiple LoRA slots and slices efficiently. + + This matches the semantics of PunicaWrapperGPU.add_lora_linear(): + - Shrink: buffer = (x @ lora_a.T) + - Expand: y += buffer @ lora_b.T + + All other functionality (weight management, TP slicing, forward logic) + is inherited from vLLM's BaseLinearLayerWithLoRA. + """ + + def create_lora_weights( + self, + max_loras: int, + lora_config, + model_config=None, + ) -> None: + super().create_lora_weights(max_loras, lora_config, model_config) + # Keep a direct reference for attribute forwarding: `base_layer` is a + # registered submodule (stored under `_modules`), so direct access via + # `object.__getattribute__` will not find it. We stash a ref in + # `__dict__` for robust lookups in `__getattr__`. + modules = object.__getattribute__(self, "_modules") + base_layer = modules.get("base_layer") or object.__getattribute__(self, "__dict__").get("base_layer") + object.__setattr__(self, "_diffusion_base_layer_ref", base_layer) + n_slices = getattr(self, "n_slices", 1) + self._diffusion_lora_active_slices = (False,) * int(n_slices) + + def reset_lora(self, index: int): + super().reset_lora(index) + n_slices = getattr(self, "n_slices", 1) + self._diffusion_lora_active_slices = (False,) * int(n_slices) + + def set_lora( + self, + index: int, + lora_a: torch.Tensor | list[torch.Tensor | None], + lora_b: torch.Tensor | list[torch.Tensor | None], + ): + super().set_lora(index, lora_a, lora_b) # type: ignore[arg-type] + + n_slices = getattr(self, "n_slices", 1) + if isinstance(lora_a, list) or isinstance(lora_b, list): + assert isinstance(lora_a, list) + assert isinstance(lora_b, list) + active_slices = [] + for a_i, b_i in zip(lora_a[:n_slices], lora_b[:n_slices]): + active_slices.append(a_i is not None and b_i is not None) + if len(active_slices) < n_slices: + active_slices.extend([False] * (n_slices - len(active_slices))) + self._diffusion_lora_active_slices = tuple(active_slices) + else: + # Single-slice layer. + self._diffusion_lora_active_slices = (True,) + + def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: + """ + override: Use simple matmul instead of punica_wrapper.add_lora_linear(). + + This matches the exact computation in PunicaWrapperGPU.add_lora_linear() + for the single-LoRA case. For packed projections (e.g. fused QKV), we + apply LoRA per-slice using `output_slices`. + """ + output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + + if not hasattr(self, "lora_a_stacked") or not hasattr(self, "lora_b_stacked"): + return output + if not self.lora_a_stacked or not self.lora_b_stacked: + return output + # Fast path: if no LoRA is active for this layer, skip matmuls. + active_slices = getattr(self, "_diffusion_lora_active_slices", None) + if active_slices is not None and not any(active_slices): + return output + + # In fully-sharded LoRA mode, vLLM uses an all-gather between shrink and + # expand for ColumnParallelLinear variants. This diffusion path doesn't + # implement that communication yet. + if getattr(self, "lora_config", None) is not None: + if self.lora_config.fully_sharded_loras and self.tp_size > 1: + raise NotImplementedError( + "Diffusion LoRA apply() does not support fully_sharded_loras with tensor parallelism yet." + ) + + original_shape = output.shape + x_flat = x.reshape(-1, x.shape[-1]) + y_flat = output.reshape(-1, output.shape[-1]) + + output_slices = getattr(self, "output_slices", None) + if output_slices is None: + # Fallback: infer slice sizes from the allocated tensors. + output_slices = tuple(lora_b.shape[2] for lora_b in self.lora_b_stacked) + + if len(output_slices) != len(self.lora_a_stacked) or len(output_slices) != len(self.lora_b_stacked): + raise RuntimeError( + "LoRA slice metadata mismatch: " + f"output_slices={len(output_slices)}, " + f"lora_a_stacked={len(self.lora_a_stacked)}, " + f"lora_b_stacked={len(self.lora_b_stacked)}" + ) + + offset = 0 + for slice_idx, slice_size in enumerate(output_slices): + if active_slices is not None and slice_idx < len(active_slices) and not active_slices[slice_idx]: + offset += slice_size + continue + + A = self.lora_a_stacked[slice_idx][0, 0, :, :] # (rank, in_dim) + B = self.lora_b_stacked[slice_idx][0, 0, :, :] # (out_dim, rank) + + if A.numel() == 0 or B.numel() == 0: + offset += slice_size + continue + + # LoRA shrink & expand as in add_lora_linear(): + # buffer = (x @ A.T) + # y += buffer @ B.T + delta = (x_flat @ A.t()) @ B.t() + y_flat[:, offset : offset + slice_size] = y_flat[:, offset : offset + slice_size] + delta + offset += slice_size + + return y_flat.view(original_shape) + + def __getattr__(self, name: str): + # The diffusion model implementations may access attributes directly + # from linear layers (e.g. QKVParallelLinear.num_heads). vLLM's LoRA + # wrappers don't forward these attributes by default, so we delegate + # missing attribute lookups to the underlying base_layer. + try: + return super().__getattr__(name) + except AttributeError as exc: + base_layer = object.__getattribute__(self, "__dict__").get("_diffusion_base_layer_ref") + if base_layer is None: + base_layer = object.__getattribute__(self, "_modules").get("base_layer") + if base_layer is None: + raise exc + try: + return getattr(base_layer, name) + except AttributeError: + raise exc diff --git a/vllm_omni/diffusion/lora/layers/column_parallel_linear.py b/vllm_omni/diffusion/lora/layers/column_parallel_linear.py new file mode 100644 index 000000000..27ac94e61 --- /dev/null +++ b/vllm_omni/diffusion/lora/layers/column_parallel_linear.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from vllm.lora.layers.column_parallel_linear import ( + ColumnParallelLinearWithLoRA, + MergedColumnParallelLinearWithLoRA, + MergedQKVParallelLinearWithLoRA, + QKVParallelLinearWithLoRA, +) + +from .base_linear import DiffusionBaseLinearLayerWithLoRA + + +class DiffusionColumnParallelLinearWithLoRA( + DiffusionBaseLinearLayerWithLoRA, + ColumnParallelLinearWithLoRA, +): + """ + Diffusion ColumnParallelLinear with LoRA. + Prioritize apply() in DiffusionBaseLinearLayerWithLoRA + """ + + pass + + +class DiffusionMergedColumnParallelLinearWithLoRA( + DiffusionBaseLinearLayerWithLoRA, + MergedColumnParallelLinearWithLoRA, +): + """ + Diffusion MergedColumnParallelLinear (gate_up_proj) with LoRA. + Prioritize apply() in DiffusionBaseLinearLayerWithLoRA + """ + + pass + + +class DiffusionQKVParallelLinearWithLoRA( + DiffusionBaseLinearLayerWithLoRA, + QKVParallelLinearWithLoRA, +): + """ + Diffusion QKVParallelLinear with single LoRA. + Prioritize apply() in DiffusionBaseLinearLayerWithLoRA + """ + + pass + + +class DiffusionMergedQKVParallelLinearWithLoRA( + DiffusionBaseLinearLayerWithLoRA, + MergedQKVParallelLinearWithLoRA, +): + """ + Diffusion MergedQKVParallelLinear (to_qkv) with 3 LoRAs. + Prioritize apply() in DiffusionBaseLinearLayerWithLoRA + """ + + pass diff --git a/vllm_omni/diffusion/lora/layers/replicated_linear.py b/vllm_omni/diffusion/lora/layers/replicated_linear.py new file mode 100644 index 000000000..e6574a04f --- /dev/null +++ b/vllm_omni/diffusion/lora/layers/replicated_linear.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from vllm.lora.layers.replicated_linear import ReplicatedLinearWithLoRA + +from .base_linear import DiffusionBaseLinearLayerWithLoRA + + +class DiffusionReplicatedLinearWithLoRA( + DiffusionBaseLinearLayerWithLoRA, + ReplicatedLinearWithLoRA, +): + """ + Diffusion ReplicatedLinear with LoRA. + Prioritize apply() in DiffusionBaseLinearLayerWithLoRA + """ + + pass diff --git a/vllm_omni/diffusion/lora/layers/row_parallel_linear.py b/vllm_omni/diffusion/lora/layers/row_parallel_linear.py new file mode 100644 index 000000000..ac2119092 --- /dev/null +++ b/vllm_omni/diffusion/lora/layers/row_parallel_linear.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from vllm.lora.layers.row_parallel_linear import RowParallelLinearWithLoRA + +from .base_linear import DiffusionBaseLinearLayerWithLoRA + + +class DiffusionRowParallelLinearWithLoRA( + DiffusionBaseLinearLayerWithLoRA, + RowParallelLinearWithLoRA, +): + """ + Diffusion RowParallelLinear with LoRA. + Prioritize apply() in DiffusionBaseLinearLayerWithLoRA + """ + + pass diff --git a/vllm_omni/diffusion/lora/manager.py b/vllm_omni/diffusion/lora/manager.py new file mode 100644 index 000000000..7fad1b9e7 --- /dev/null +++ b/vllm_omni/diffusion/lora/manager.py @@ -0,0 +1,631 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import time +from collections import OrderedDict + +import torch +import torch.nn as nn +from vllm.logger import init_logger +from vllm.lora.layers import BaseLayerWithLoRA +from vllm.lora.lora_model import LoRAModel +from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights +from vllm.lora.peft_helper import PEFTHelper +from vllm.lora.request import LoRARequest +from vllm.lora.utils import ( + get_adapter_absolute_path, + get_supported_lora_modules, + replace_submodule, +) +from vllm.model_executor.layers.linear import MergedColumnParallelLinear, QKVParallelLinear + +from vllm_omni.config.lora import LoRAConfig +from vllm_omni.diffusion.lora.utils import ( + _expand_expected_modules_for_packed_layers, + _match_target_modules, + from_layer_diffusion, +) +from vllm_omni.lora.utils import stable_lora_int_id + +logger = init_logger(__name__) + + +class DiffusionLoRAManager: + """Manager for LoRA adapters in diffusion models. + + Reuses vLLM's LoRA infrastructure, adapted for diffusion pipelines. + Uses LRU cache management similar to LRUCacheLoRAModelManager. + """ + + def __init__( + self, + pipeline: nn.Module, + device: torch.device, + dtype: torch.dtype, + max_cached_adapters: int = 1, + lora_path: str | None = None, + lora_scale: float = 1.0, + ): + """ + Initialize the DiffusionLoRAManager. + + Args: + max_cached_adapters: Maximum number of LoRA adapters to keep in the + CPU-side cache (LRU). This mirrors vLLM's `max_cpu_loras` and is + exposed to users via `OmniDiffusionConfig.max_cpu_loras`. + """ + self.pipeline = pipeline + self.device = device + self.dtype = dtype + + # Cache supported/expected module suffixes once, before any layer + # replacement happens. After LoRA layers are injected, the original + # LinearBase layers become submodules named "*.base_layer", and calling + # vLLM's get_supported_lora_modules() again would incorrectly yield + # "base_layer" instead of the real target module suffixes. + self._supported_lora_modules = self._compute_supported_lora_modules() + self._packed_modules_mapping = self._compute_packed_modules_mapping() + self._expected_lora_modules = _expand_expected_modules_for_packed_layers( + self._supported_lora_modules, + self._packed_modules_mapping, + ) + + # LRU-style cache management + self.max_cached_adapters = max_cached_adapters # max_cpu_loras + self._registered_adapters: dict[int, LoRAModel] = {} # adapter_id -> LoRAModel + self._active_adapter_id: int | None = None + self._adapter_scales: dict[int, float] = {} # adapter_id -> external scale + + # LRU cache tracking (adapter_id -> last_used_time) + self._adapter_access_order: OrderedDict[int, float] = OrderedDict() + # Pinned adapters are not evicted + self._pinned_adapters: set[int] = set() + + # track replaced modules + # key: full module name (component.module.path); value: LoRA layer + self._lora_modules: dict[str, BaseLayerWithLoRA] = {} + # Track the maximum LoRA rank we've allocated buffers for. + self._max_lora_rank: int = 0 + + logger.info( + "Initializing DiffusionLoRAManager: device=%s, dtype=%s, max_cached_adapters=%d, static_lora_path=%s", + device, + dtype, + max_cached_adapters, + lora_path, + ) + + if lora_path is not None: + logger.info("Loading LoRA during initialization from %s with scale %.2f", lora_path, lora_scale) + init_request = LoRARequest( + lora_name="static", + lora_int_id=stable_lora_int_id(lora_path), + lora_path=lora_path, + ) + self.set_active_adapter(init_request, lora_scale) + + def _compute_supported_lora_modules(self) -> set[str]: + """Compute supported LoRA module suffixes for this pipeline. + + vLLM's get_supported_lora_modules() returns suffixes for LinearBase + modules. After this manager replaces layers with BaseLayerWithLoRA + wrappers, those LinearBase modules become nested under ".base_layer", + which would cause get_supported_lora_modules() to return "base_layer". + To make adapter loading stable across multiple adapters, we also accept + suffixes from existing BaseLayerWithLoRA wrappers and drop "base_layer" + when appropriate. + """ + supported = set(get_supported_lora_modules(self.pipeline)) + + has_lora_wrappers = False + for name, module in self.pipeline.named_modules(): + if isinstance(module, BaseLayerWithLoRA): + has_lora_wrappers = True + supported.add(name.split(".")[-1]) + + if has_lora_wrappers: + supported.discard("base_layer") + + return supported + + def _compute_packed_modules_mapping(self) -> dict[str, list[str]]: + """Collect packed->sublayer mappings from the diffusion model. + + vLLM models declare `packed_modules_mapping` on the model class. For + diffusion pipelines, we attach the same mapping on the transformer + module(s) that implement packed (fused) projections, so LoRA loading can + accept checkpoints trained against the logical sub-projections. + """ + mapping: dict[str, list[str]] = {} + for module in self.pipeline.modules(): + packed = getattr(module, "packed_modules_mapping", None) + if not isinstance(packed, dict): + continue + for packed_name, sub_names in packed.items(): + if not isinstance(packed_name, str) or not packed_name: + continue + if not isinstance(sub_names, (list, tuple)) or not all(isinstance(s, str) for s in sub_names): + continue + sub_names_list = list(sub_names) + if not sub_names_list: + continue + + existing = mapping.get(packed_name) + if existing is None: + mapping[packed_name] = sub_names_list + elif existing != sub_names_list: + logger.warning( + "Conflicting packed_modules_mapping for %s: %s vs %s; using %s", + packed_name, + existing, + sub_names_list, + existing, + ) + + return mapping + + def _get_packed_sublayer_suffixes(self, packed_module_suffix: str, n_slices: int) -> list[str] | None: + sub_suffixes = self._packed_modules_mapping.get(packed_module_suffix) + if not sub_suffixes: + return None + if len(sub_suffixes) != n_slices: + logger.warning( + "packed_modules_mapping[%s] has %d slices but layer expects %d; skipping sublayer lookup", + packed_module_suffix, + len(sub_suffixes), + n_slices, + ) + return None + return sub_suffixes + + def set_active_adapter(self, lora_request: LoRARequest | None, lora_scale: float = 1.0) -> None: + """Set the active LoRA adapter for the pipeline. + + Args: + lora_request: The LoRA request, or None to deactivate all adapters. + lora_scale: The external scale for the LoRA adapter. + """ + if lora_request is None: + logger.debug("No lora_request provided, deactivating all LoRA adapters") + self._deactivate_all_adapters() + return + + adapter_id = lora_request.lora_int_id + logger.debug( + "Setting active adapter: id=%d, name=%s, path=%s, scale=%.2f, cache_size=%d/%d", + adapter_id, + lora_request.lora_name, + lora_request.lora_path, + lora_scale, + len(self._registered_adapters), + self.max_cached_adapters, + ) + if adapter_id not in self._registered_adapters: + logger.info("Loading new adapter: id=%d, name=%s", adapter_id, lora_request.lora_name) + self.add_adapter(lora_request, lora_scale) + else: + logger.debug("Adapter %d already loaded, activating", adapter_id) + + # update access order + self._adapter_scales[adapter_id] = lora_scale + self._adapter_access_order[adapter_id] = time.time() + self._adapter_access_order.move_to_end(adapter_id) + + self._activate_adapter(adapter_id) + + def _load_adapter( + self, + lora_request: LoRARequest, + ) -> tuple[LoRAModel, PEFTHelper]: + if not self._expected_lora_modules: + raise ValueError("No supported LoRA modules found in the diffusion pipeline.") + + logger.debug("Supported LoRA modules: %s", self._expected_lora_modules) + + lora_path = get_adapter_absolute_path(lora_request.lora_path) + logger.debug("Resolved LoRA path: %s", lora_path) + + peft_helper = PEFTHelper.from_local_dir( + lora_path, + max_position_embeddings=None, # no need in diffusion + tensorizer_config_dict=lora_request.tensorizer_config_dict, + ) + + logger.info( + "Loaded PEFT config: r=%d, lora_alpha=%d, target_modules=%s", + peft_helper.r, + peft_helper.lora_alpha, + peft_helper.target_modules, + ) + + lora_model = LoRAModel.from_local_checkpoint( + lora_path, + expected_lora_modules=self._expected_lora_modules, + peft_helper=peft_helper, + lora_model_id=lora_request.lora_int_id, + device="cpu", # consistent w/ vllm's behavior + dtype=self.dtype, + model_vocab_size=None, + tensorizer_config_dict=lora_request.tensorizer_config_dict, + weights_mapper=None, + ) + + logger.info( + "Loaded LoRA model: id=%d, num_modules=%d, modules=%s", + lora_model.id, + len(lora_model.loras), + list(lora_model.loras.keys()), + ) + + for lora in lora_model.loras.values(): + lora.optimize() # ref: _create_merged_loras_inplace, internal scaling + + return lora_model, peft_helper + + def _get_packed_modules_list(self, module: nn.Module) -> list[str]: + """Return a packed_modules_list suitable for vLLM LoRA can_replace_layer(). + + Diffusion transformers frequently use packed projection layers like + QKVParallelLinear (fused QKV). vLLM's LoRA replacement logic relies on + `packed_modules_list` length to decide between single-slice vs packed + LoRA layer implementations. + """ + if isinstance(module, QKVParallelLinear): + # Treat diffusion QKV as a 3-slice packed projection by default. + return ["q", "k", "v"] + if isinstance(module, MergedColumnParallelLinear): + # 2-slice packed projection (e.g. fused MLP projections). + return ["0", "1"] + return [] + + def _replace_layers_with_lora(self, peft_helper: PEFTHelper) -> None: + self._ensure_max_lora_rank(peft_helper.r) + + target_modules = getattr(peft_helper, "target_modules", None) + target_modules_list: list[str] | None = None + target_modules_pattern: str | None = None + if isinstance(target_modules, str) and target_modules: + target_modules_pattern = target_modules + elif isinstance(target_modules, list) and target_modules: + target_modules_list = target_modules + + def _matches_target(module_name: str) -> bool: + if target_modules_pattern is not None: + import regex as re + + return re.search(target_modules_pattern, module_name) is not None + if target_modules_list is None: + return True + return _match_target_modules(module_name, target_modules_list) + + # dummy lora config + lora_config = LoRAConfig( + max_lora_rank=self._max_lora_rank, + max_loras=1, + max_cpu_loras=self.max_cached_adapters, + lora_dtype=self.dtype, + fully_sharded_loras=False, + ) + + for component_name in ("transformer", "transformer_2", "dit"): + if not hasattr(self.pipeline, component_name): + continue + component = getattr(self.pipeline, component_name) + if not isinstance(component, nn.Module): + continue + + for module_name, module in component.named_modules(remove_duplicate=False): + # Don't recurse into already-replaced LoRA wrappers. Their + # original LinearBase lives under "base_layer", and replacing + # that again would nest LoRA wrappers and break execution. + if isinstance(module, BaseLayerWithLoRA) or "base_layer" in module_name.split("."): + continue + + full_module_name = f"{component_name}.{module_name}" + if full_module_name in self._lora_modules: + logger.debug("Layer %s already replaced, skipping", full_module_name) + continue + + packed_modules_list = self._get_packed_modules_list(module) + if target_modules_pattern is not None or target_modules_list is not None: + should_replace = _matches_target(full_module_name) + if not should_replace and len(packed_modules_list) > 1: + prefix, _, packed_suffix = full_module_name.rpartition(".") + sub_suffixes = self._get_packed_sublayer_suffixes(packed_suffix, len(packed_modules_list)) + if sub_suffixes is not None: + for sub_suffix in sub_suffixes: + sub_full_name = f"{prefix}.{sub_suffix}" if prefix else sub_suffix + if _matches_target(sub_full_name): + should_replace = True + break + + if not should_replace: + continue + + lora_layer = from_layer_diffusion( + layer=module, + max_loras=1, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=None, + ) + + if lora_layer is not module and isinstance(lora_layer, BaseLayerWithLoRA): + replace_submodule(component, module_name, lora_layer) + self._lora_modules[full_module_name] = lora_layer + logger.debug("Replaced layer: %s -> %s", full_module_name, type(lora_layer).__name__) + + def _ensure_max_lora_rank(self, min_rank: int) -> None: + """Ensure LoRA buffers can accommodate adapters up to `min_rank`. + + We allocate per-layer LoRA buffers once when we first replace layers. + If a later adapter has a larger rank, we need to reinitialize those + buffers and re-apply the currently active adapter. + """ + if min_rank <= self._max_lora_rank: + return + + if min_rank <= 0: + raise ValueError(f"Invalid LoRA rank: {min_rank}") + + logger.info("Increasing max LoRA rank: %d -> %d", self._max_lora_rank, min_rank) + self._max_lora_rank = min_rank + + if not self._lora_modules: + return + + lora_config = LoRAConfig( + max_lora_rank=self._max_lora_rank, + max_loras=1, + max_cpu_loras=self.max_cached_adapters, + lora_dtype=self.dtype, + fully_sharded_loras=False, + ) + + # Recreate per-layer buffers with the new maximum rank. + for lora_layer in self._lora_modules.values(): + lora_layer.create_lora_weights(max_loras=1, lora_config=lora_config, model_config=None) + + # Re-apply active adapter if needed (buffers were reset). + if self._active_adapter_id is not None: + active_id = self._active_adapter_id + self._active_adapter_id = None + self._activate_adapter(active_id) + + def _get_lora_weights( + self, + lora_model: LoRAModel, + full_module_name: str, + ) -> LoRALayerWeights | PackedLoRALayerWeights | None: + """Best-effort lookup for LoRA weights by name. + + Tries: + - Full module name (e.g. transformer.blocks.0.attn.to_qkv) + - Relative name without the top-level component (e.g. blocks.0.attn.to_qkv) + - Suffix-only name (e.g. to_qkv) + """ + lora_weights = lora_model.get_lora(full_module_name) + if lora_weights is not None: + return lora_weights + + component_relative_name = full_module_name.split(".", 1)[-1] if "." in full_module_name else full_module_name + lora_weights = lora_model.get_lora(component_relative_name) + if lora_weights is not None: + return lora_weights + + module_suffix = full_module_name.split(".")[-1] + return lora_model.get_lora(module_suffix) + + def _activate_adapter(self, adapter_id: int) -> None: + if self._active_adapter_id == adapter_id: + logger.debug("Adapter %d already active, skipping", adapter_id) + return + + logger.info("Activating adapter: id=%d", adapter_id) + lora_model = self._registered_adapters[adapter_id] + + # activate weights in each LoRA layer + for full_module_name, lora_layer in self._lora_modules.items(): + lora_weights = self._get_lora_weights(lora_model, full_module_name) + + if lora_weights is None: + n_slices = getattr(lora_layer, "n_slices", 1) + if n_slices > 1: + prefix, _, packed_suffix = full_module_name.rpartition(".") + sub_suffixes = self._get_packed_sublayer_suffixes(packed_suffix, n_slices) + if sub_suffixes is None: + lora_layer.reset_lora(0) + continue + + sub_loras: list[LoRALayerWeights | None] = [] + any_found = False + for sub_suffix in sub_suffixes: + sub_full_name = f"{prefix}.{sub_suffix}" if prefix else sub_suffix + sub_lora = self._get_lora_weights(lora_model, sub_full_name) + if sub_lora is not None: + any_found = True + # Packed layers expect plain (non-packed) subloras. + if isinstance(sub_lora, PackedLoRALayerWeights): + sub_lora = None + sub_loras.append(sub_lora if isinstance(sub_lora, LoRALayerWeights) else None) + + if not any_found: + lora_layer.reset_lora(0) + continue + + scale = self._adapter_scales.get(adapter_id, 1.0) + lora_a_list: list[torch.Tensor | None] = [] + lora_b_list: list[torch.Tensor | None] = [] + for sub_lora in sub_loras: + if sub_lora is None: + lora_a_list.append(None) + lora_b_list.append(None) + continue + lora_a_list.append(sub_lora.lora_a) + lora_b_list.append(sub_lora.lora_b * scale) + + lora_layer.set_lora(index=0, lora_a=lora_a_list, lora_b=lora_b_list) + logger.debug( + "Activated packed LoRA for %s via submodules=%s (scale=%.2f)", + full_module_name, + sub_suffixes, + scale, + ) + else: + lora_layer.reset_lora(0) + continue + + scale = self._adapter_scales.get(adapter_id, 1.0) + + # Packed LoRA weights already provide per-slice tensors. + if isinstance(lora_weights, PackedLoRALayerWeights): + lora_a_list = lora_weights.lora_a + lora_b_list = [ + None if b is None else b * scale # type: ignore[operator] + for b in lora_weights.lora_b + ] + lora_layer.set_lora(index=0, lora_a=lora_a_list, lora_b=lora_b_list) + logger.debug( + "Activated packed LoRA for %s (scale=%.2f)", + full_module_name, + scale, + ) + continue + + # Fused (non-packed) weights: if the layer is multi-slice, split B. + n_slices = getattr(lora_layer, "n_slices", 1) + if n_slices > 1: + output_slices = getattr(lora_layer, "output_slices", None) + if output_slices is None: + lora_layer.reset_lora(0) + continue + + total = sum(output_slices) + if lora_weights.lora_b.shape[0] != total: + logger.warning( + "Skipping LoRA for %s due to shape mismatch: lora_b[0]=%d != sum(output_slices)=%d", + full_module_name, + lora_weights.lora_b.shape[0], + total, + ) + lora_layer.reset_lora(0) + continue + + b_splits = list(torch.split(lora_weights.lora_b, list(output_slices), dim=0)) + lora_a_list = [lora_weights.lora_a] * n_slices + lora_b_list = [b * scale for b in b_splits] + lora_layer.set_lora(index=0, lora_a=lora_a_list, lora_b=lora_b_list) + logger.debug( + "Activated fused LoRA for packed layer %s (scale=%.2f)", + full_module_name, + scale, + ) + continue + + scaled_lora_b = lora_weights.lora_b * scale + lora_layer.set_lora(index=0, lora_a=lora_weights.lora_a, lora_b=scaled_lora_b) + logger.debug( + "Activated LoRA for %s: lora_a shape=%s, lora_b shape=%s, scale=%.2f", + full_module_name, + lora_weights.lora_a.shape, + lora_weights.lora_b.shape, + scale, + ) + + self._active_adapter_id = adapter_id + + def _deactivate_all_adapters(self) -> None: + logger.info("Deactivating all adapters: %d layers", len(self._lora_modules)) + for lora_layer in self._lora_modules.values(): + lora_layer.reset_lora(0) + self._active_adapter_id = None + logger.debug("All adapters deactivated") + + def _evict_if_needed(self) -> None: + while len(self._registered_adapters) > self.max_cached_adapters: + # Pick LRU among non-pinned adapters + evict_candidates = [aid for aid in self._adapter_access_order.keys() if aid not in self._pinned_adapters] + if not evict_candidates: + logger.warning( + "Cache full (%d) but all adapters are pinned; cannot evict. " + "Increase max_cached_adapters or unpin adapters.", + self.max_cached_adapters, + ) + break + + lru_adapter_id = evict_candidates[0] + logger.info( + "Evicting LRU adapter: id=%d (cache: %d/%d)", + lru_adapter_id, + len(self._registered_adapters), + self.max_cached_adapters, + ) + self.remove_adapter(lru_adapter_id) + + def add_adapter(self, lora_request: LoRARequest, lora_scale: float = 1.0) -> bool: + """ + Add a new adapter to the cache without activating it. + """ + adapter_id = lora_request.lora_int_id + + if adapter_id in self._registered_adapters: + logger.debug("Adapter %d already registered, skipping", adapter_id) + return False + + logger.info("Adding new adapter: id=%d, name=%s", adapter_id, lora_request.lora_name) + lora_model, peft_helper = self._load_adapter(lora_request) + self._registered_adapters[adapter_id] = lora_model + self._adapter_scales[adapter_id] = lora_scale + + self._replace_layers_with_lora(peft_helper) + + self._adapter_access_order[adapter_id] = time.time() + self._adapter_access_order.move_to_end(adapter_id) + + # evict if cache full + self._evict_if_needed() + + logger.debug( + "Adapter %d added, cache size: %d/%d", adapter_id, len(self._registered_adapters), self.max_cached_adapters + ) + return True + + def remove_adapter(self, adapter_id: int) -> bool: + """ + Remove an adapter from the cache. + """ + if adapter_id not in self._registered_adapters: + logger.debug("Adapter %d not found, cannot remove", adapter_id) + return False + + logger.info("Removing adapter: id=%d", adapter_id) + if self._active_adapter_id == adapter_id: + self._deactivate_all_adapters() + + del self._registered_adapters[adapter_id] + self._adapter_scales.pop(adapter_id, None) + self._adapter_access_order.pop(adapter_id, None) + self._pinned_adapters.discard(adapter_id) + logger.debug( + "Adapter %d removed, cache size: %d/%d", + adapter_id, + len(self._registered_adapters), + self.max_cached_adapters, + ) + return True + + def list_adapters(self) -> list[int]: + """Return list of registered adapter ids.""" + return list(self._registered_adapters.keys()) + + def pin_adapter(self, adapter_id: int) -> bool: + """Mark an adapter as pinned so it will not be evicted.""" + if adapter_id not in self._registered_adapters: + logger.debug("Adapter %d not found, cannot pin", adapter_id) + return False + self._pinned_adapters.add(adapter_id) + # Touch access order so it is most recently used + self._adapter_access_order[adapter_id] = time.time() + self._adapter_access_order.move_to_end(adapter_id) + logger.info("Pinned adapter id=%d (won't be evicted)", adapter_id) + return True diff --git a/vllm_omni/diffusion/lora/utils.py b/vllm_omni/diffusion/lora/utils.py new file mode 100644 index 000000000..5f1baea34 --- /dev/null +++ b/vllm_omni/diffusion/lora/utils.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import torch.nn as nn +from transformers import PretrainedConfig + +from vllm_omni.config.lora import LoRAConfig +from vllm_omni.diffusion.lora.layers import ( + DiffusionColumnParallelLinearWithLoRA, + DiffusionMergedColumnParallelLinearWithLoRA, + DiffusionMergedQKVParallelLinearWithLoRA, + DiffusionQKVParallelLinearWithLoRA, + DiffusionReplicatedLinearWithLoRA, + DiffusionRowParallelLinearWithLoRA, +) + + +def _match_target_modules(module_name: str, target_modules: list[str]) -> bool: + """from vllm/lora/model_manager.py _match_target_modules, helper function""" + import regex as re + + return any( + re.match(rf".*\.{target_module}$", module_name) or target_module == module_name + for target_module in target_modules + ) + + +def _expand_expected_modules_for_packed_layers( + supported_modules: set[str], + packed_modules_mapping: dict[str, list[str]] | None, +) -> set[str]: + """Expand expected LoRA module suffixes for packed (fused) projections. + + Some diffusion models use packed projections like `to_qkv` or `w13`, while + LoRA checkpoints are typically saved against the logical sub-projections + (e.g. `to_q`/`to_k`/`to_v`, `w1`/`w3`). The packed layer name is present in + `supported_modules`, but the sublayer names are not. Expanding the set + ensures these sublayer keys are not dropped when loading a LoRA checkpoint. + + The packed→sublayer mapping is model-specific (see each diffusion model's + `packed_modules_mapping`) so new packed layers are added alongside the model + implementation rather than hard-coded in the LoRA framework. + """ + expanded = set(supported_modules) + if not packed_modules_mapping: + return expanded + + for packed_name, sub_names in packed_modules_mapping.items(): + if packed_name in supported_modules: + expanded.update(sub_names) + + return expanded + + +def from_layer_diffusion( + layer: nn.Module, + max_loras: int, + lora_config: LoRAConfig, + packed_modules_list: list[str], + model_config: PretrainedConfig | None = None, +) -> nn.Module: + """ + Diffusion-specific layer replacement. similar to vLLM's `from_layer` + """ + diffusion_lora_classes = [ + DiffusionMergedQKVParallelLinearWithLoRA, + DiffusionQKVParallelLinearWithLoRA, + DiffusionMergedColumnParallelLinearWithLoRA, + DiffusionColumnParallelLinearWithLoRA, + DiffusionRowParallelLinearWithLoRA, + DiffusionReplicatedLinearWithLoRA, + ] + + for lora_cls in diffusion_lora_classes: + if lora_cls.can_replace_layer( + source_layer=layer, + lora_config=lora_config, + packed_modules_list=packed_modules_list, + model_config=model_config, + ): + instance = lora_cls(layer) # type: ignore[arg-type] + instance.create_lora_weights(max_loras, lora_config, model_config) + return instance + + return layer diff --git a/vllm_omni/diffusion/model_loader/diffusers_loader.py b/vllm_omni/diffusion/model_loader/diffusers_loader.py index 0398bfbc2..892954ce9 100644 --- a/vllm_omni/diffusion/model_loader/diffusers_loader.py +++ b/vllm_omni/diffusion/model_loader/diffusers_loader.py @@ -94,6 +94,7 @@ class DiffusersPipelineLoader: load_format = self.load_config.load_format use_safetensors = False index_file = DIFFUSION_MODEL_WEIGHTS_INDEX + index_file_with_subfolder = f"{subfolder}/{index_file}" if subfolder else index_file # only hf is supported currently if load_format == "auto": @@ -129,8 +130,8 @@ class DiffusersPipelineLoader: for pattern in allow_patterns: hf_weights_files += glob.glob(os.path.join(hf_folder, pattern)) if len(hf_weights_files) > 0: - if pattern == "*.safetensors": - use_safetensors = True + # Decide by actual files rather than pattern name (patterns may include subfolders). + use_safetensors = any(f.endswith(".safetensors") for f in hf_weights_files) break if use_safetensors: @@ -142,11 +143,22 @@ class DiffusersPipelineLoader: if not is_local: download_safetensors_index_file_from_hf( model_name_or_path, - index_file, + index_file_with_subfolder, self.load_config.download_dir, revision, ) - hf_weights_files = filter_duplicate_safetensors_files(hf_weights_files, hf_folder, index_file) + # Some diffusers pipelines keep component weights under a + # subfolder (e.g. "transformer/") and the corresponding index file + # uses filenames relative to that subfolder. vLLM's + # `filter_duplicate_safetensors_files` expects weight_map entries + # to be relative to the `hf_folder` we pass in, so we point it to + # the component subfolder to avoid filtering out all shards. + filter_folder = os.path.join(hf_folder, subfolder) if subfolder is not None else hf_folder + hf_weights_files = filter_duplicate_safetensors_files( + hf_weights_files, + filter_folder, + index_file, + ) else: hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files) @@ -188,8 +200,9 @@ class DiffusersPipelineLoader: def download_model(self, model_config: ModelConfig) -> None: self._prepare_weights( - model_config.model, - model_config.revision, + model_name_or_path=model_config.model, + subfolder=None, + revision=model_config.revision, fall_back_to_pt=True, allow_patterns_overrides=None, ) diff --git a/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py b/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py index 86658a01d..312ac9b6a 100644 --- a/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py +++ b/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py @@ -518,6 +518,10 @@ class Flux2Transformer2DModel(nn.Module): """ _repeated_blocks = ["Flux2TransformerBlock", "Flux2SingleTransformerBlock"] + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + "add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"], + } def __init__( self, diff --git a/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py b/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py index 615b9194a..c85c0bfd9 100644 --- a/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py +++ b/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py @@ -551,6 +551,10 @@ class GlmImageTransformer2DModel(CachedTransformer): are read from `od_config.tf_model_config`. """ + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + } + def __init__( self, od_config: OmniDiffusionConfig, diff --git a/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py b/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py index d2a55046d..6bb282a80 100644 --- a/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py +++ b/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py @@ -503,6 +503,11 @@ class LongCatImageTransformer2DModel(nn.Module): Supports Sequence Parallelism (Ulysses and Ring) when configured via OmniDiffusionConfig. """ + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + "add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"], + } + def __init__( self, od_config: OmniDiffusionConfig, diff --git a/vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py b/vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py index 06bbf5706..ae6cf6b0c 100644 --- a/vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py +++ b/vllm_omni/diffusion/models/ovis_image/ovis_image_transformer.py @@ -366,6 +366,10 @@ class OvisImageTransformer2DModel(nn.Module): """ _repeated_blocks = ["OvisImageTransformerBlock", "OvisImageSingleTransformerBlock"] + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + "add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"], + } def __init__( self, diff --git a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py index 14905e2a8..04890fbb9 100644 --- a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py +++ b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py @@ -782,6 +782,10 @@ class QwenImageTransformer2DModel(CachedTransformer): # -- typically a transformer layer # used for torch compile optimizations _repeated_blocks = ["QwenImageTransformerBlock"] + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + "add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"], + } # Sequence Parallelism plan (following diffusers' _cp_plan pattern) # Similar to Z-Image's UnifiedPrepare, we use ImageRopePrepare to create diff --git a/vllm_omni/diffusion/models/sd3/sd3_transformer.py b/vllm_omni/diffusion/models/sd3/sd3_transformer.py index 22a11741a..e60bcbe5a 100644 --- a/vllm_omni/diffusion/models/sd3/sd3_transformer.py +++ b/vllm_omni/diffusion/models/sd3/sd3_transformer.py @@ -322,6 +322,10 @@ class SD3Transformer2DModel(nn.Module): """ _repeated_blocks = ["SD3TransformerBlock"] + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + "add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"], + } def __init__( self, diff --git a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py index a1e2d789b..f56833e48 100644 --- a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py +++ b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py @@ -516,6 +516,9 @@ class WanTransformer3DModel(nn.Module): """ _repeated_blocks = ["WanTransformerBlock"] + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + } def __init__( self, diff --git a/vllm_omni/diffusion/models/z_image/z_image_transformer.py b/vllm_omni/diffusion/models/z_image/z_image_transformer.py index 247d914e9..2b7d4eb5b 100644 --- a/vllm_omni/diffusion/models/z_image/z_image_transformer.py +++ b/vllm_omni/diffusion/models/z_image/z_image_transformer.py @@ -544,6 +544,10 @@ class ZImageTransformer2DModel(CachedTransformer): """ _repeated_blocks = ["ZImageTransformerBlock"] + packed_modules_mapping = { + "to_qkv": ["to_q", "to_k", "to_v"], + "w13": ["w1", "w3"], + } # Sequence Parallelism for Z-Image (following diffusers' _cp_plan pattern) # Similar to how Wan uses `rope` module's split_output to shard rotary embeddings, diff --git a/vllm_omni/diffusion/request.py b/vllm_omni/diffusion/request.py index 279b3bf1d..89c0a79f1 100644 --- a/vllm_omni/diffusion/request.py +++ b/vllm_omni/diffusion/request.py @@ -9,6 +9,8 @@ from typing import Any import PIL.Image import torch +from vllm_omni.lora.request import LoRARequest + @dataclass class OmniDiffusionRequest: @@ -142,6 +144,10 @@ class OmniDiffusionRequest: save_output: bool = True return_frames: bool = False + # LoRA + lora_request: LoRARequest | None = None + lora_scale: float = 1.0 + # STA parameters STA_param: list | None = None is_cfg_negative: bool = False diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py b/vllm_omni/diffusion/worker/gpu_diffusion_worker.py index 90d169a0e..cadb6b64d 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py +++ b/vllm_omni/diffusion/worker/gpu_diffusion_worker.py @@ -28,9 +28,11 @@ from vllm_omni.diffusion.distributed.parallel_state import ( initialize_model_parallel, ) from vllm_omni.diffusion.forward_context import set_forward_context +from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager from vllm_omni.diffusion.profiler import CurrentProfiler from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner +from vllm_omni.lora.request import LoRARequest logger = init_logger(__name__) @@ -61,6 +63,7 @@ class GPUDiffusionWorker: self.vllm_config: VllmConfig | None = None self.model_runner: GPUDiffusionModelRunner | None = None self._sleep_saved_buffers: dict[str, torch.Tensor] = {} + self.lora_manager: DiffusionLoRAManager | None = None self.init_device() def init_device(self) -> None: @@ -110,6 +113,15 @@ class GPUDiffusionWorker: self.model_runner.load_model( memory_pool_context_fn=self._maybe_get_memory_pool_context, ) + assert self.model_runner.pipeline is not None + self.lora_manager = DiffusionLoRAManager( + pipeline=self.model_runner.pipeline, + device=self.device, + dtype=self.od_config.dtype, + max_cached_adapters=self.od_config.max_cpu_loras, + lora_path=self.od_config.lora_path, + lora_scale=self.od_config.lora_scale, + ) logger.info(f"Worker {self.rank}: Initialization complete.") def generate(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput: @@ -129,6 +141,33 @@ class GPUDiffusionWorker: def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput: """Execute a forward pass by delegating to the model runner.""" assert self.model_runner is not None, "Model runner not initialized" + if self.lora_manager is not None and reqs: + req = reqs[0] + + if len(reqs) > 1: + # This worker (and the current diffusion model runner) applies + # a single LoRA to the whole batch. Reject inconsistent LoRA + # settings to avoid silently applying the wrong adapter. + def _lora_key(r: OmniDiffusionRequest): + if r.lora_request is None: + return None + lr = r.lora_request + return (lr.lora_name, lr.lora_int_id, lr.lora_path, lr.tensorizer_config_dict) + + key0 = _lora_key(req) + scale0 = req.lora_scale if key0 is not None else None + for other in reqs[1:]: + if _lora_key(other) != key0: + raise ValueError("All requests in a diffusion batch must share the same LoRARequest.") + if key0 is not None and other.lora_scale != scale0: + raise ValueError("All requests in a diffusion batch must share the same lora_scale.") + + try: + self.lora_manager.set_active_adapter(req.lora_request, req.lora_scale) + except Exception as exc: + if req.lora_request is not None: + raise + logger.warning("LoRA activation skipped: %s", exc) return self.model_runner.execute_model(reqs) def load_weights(self, weights) -> set[str]: @@ -136,6 +175,18 @@ class GPUDiffusionWorker: assert self.model_runner is not None, "Model runner not initialized" return self.model_runner.load_weights(weights) + def remove_lora(self, adapter_id: int) -> bool: + return self.lora_manager.remove_adapter(adapter_id) + + def add_lora(self, lora_request: LoRARequest, lora_scale: float = 1.0) -> bool: + return self.lora_manager.add_adapter(lora_request, lora_scale) + + def list_loras(self) -> list[int]: + return self.lora_manager.list_adapters() + + def pin_lora(self, adapter_id: int) -> bool: + return self.lora_manager.pin_adapter(adapter_id) + def sleep(self, level: int = 1) -> bool: """ Put the worker to sleep, offloading model weights. diff --git a/vllm_omni/engine/input_processor.py b/vllm_omni/engine/input_processor.py index eb81f38dc..91e12f6a6 100644 --- a/vllm_omni/engine/input_processor.py +++ b/vllm_omni/engine/input_processor.py @@ -7,7 +7,6 @@ from vllm.config import VllmConfig from vllm.inputs import ProcessorInputs, PromptType from vllm.inputs.parse import split_enc_dec_inputs from vllm.logger import init_logger -from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict from vllm.multimodal.utils import argsort_mm_positions @@ -25,6 +24,7 @@ from vllm_omni.engine import ( PromptEmbedsPayload, ) from vllm_omni.inputs.preprocess import OmniInputPreprocessor +from vllm_omni.lora.request import LoRARequest logger = init_logger(__name__) diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 88a290cdf..3dd687a8b 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -11,13 +11,11 @@ from typing import Any from vllm.config import VllmConfig from vllm.inputs.preprocess import InputPreprocessor from vllm.logger import init_logger -from vllm.lora.request import LoRARequest from vllm.plugins.io_processors import get_io_processor from vllm.sampling_params import SamplingParams from vllm.tokenizers import TokenizerLike from vllm.v1.engine.exceptions import EngineDeadError -# Internal imports (our code) from vllm_omni.config import OmniModelConfig from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.distributed.omni_connectors.adapter import compute_talker_prompt_ids_length, try_send_via_connector @@ -34,6 +32,9 @@ from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load from vllm_omni.entrypoints.utils import ( get_final_stage_id_for_e2e, ) + +# Internal imports (our code) +from vllm_omni.lora.request import LoRARequest from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) diff --git a/vllm_omni/entrypoints/async_omni_diffusion.py b/vllm_omni/entrypoints/async_omni_diffusion.py index bfaf7aaae..6d34e3445 100644 --- a/vllm_omni/entrypoints/async_omni_diffusion.py +++ b/vllm_omni/entrypoints/async_omni_diffusion.py @@ -22,6 +22,7 @@ from vllm.transformers_utils.config import get_hf_file_to_dict from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig from vllm_omni.diffusion.diffusion_engine import DiffusionEngine from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.lora.request import LoRARequest from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) @@ -154,6 +155,7 @@ class AsyncOmniDiffusion: negative_prompt: str | None = None, num_outputs_per_prompt: int = 1, seed: int | None = None, + lora_request=None, **kwargs: Any, ) -> OmniRequestOutput: """Generate images asynchronously from a text prompt. @@ -189,6 +191,7 @@ class AsyncOmniDiffusion: "negative_prompt": negative_prompt, "num_outputs_per_prompt": num_outputs_per_prompt, "seed": seed, + "lora_request": lora_request, **kwargs, } if guidance_scale is not None: @@ -305,3 +308,65 @@ class AsyncOmniDiffusion: def is_stopped(self) -> bool: """Check if the engine is stopped.""" return self._closed + + async def remove_lora(self, adapter_id: int) -> bool: + """Remove a LoRA""" + loop = asyncio.get_event_loop() + results = await loop.run_in_executor( + self._executor, + self.engine.collective_rpc, + "remove_lora", + None, + (adapter_id,), + {}, + None, + ) + return all(results) if isinstance(results, list) else results + + async def add_lora(self, lora_request: LoRARequest, lora_scale: float = 1.0) -> bool: + """Add a LoRA adapter""" + loop = asyncio.get_event_loop() + results = await loop.run_in_executor( + self._executor, + self.engine.collective_rpc, + "add_lora", + None, + (), + {"lora_request": lora_request, "lora_scale": lora_scale}, + None, + ) + return all(results) if isinstance(results, list) else results + + async def list_loras(self) -> list[int]: + """List all registered LoRA adapter IDs.""" + loop = asyncio.get_event_loop() + results = await loop.run_in_executor( + self._executor, + self.engine.collective_rpc, + "list_loras", + None, + (), + {}, + None, + ) + # collective_rpc returns list from workers; flatten unique ids + if not isinstance(results, list): + return results or [] + merged: set[int] = set() + for part in results: + merged.update(part or []) + return sorted(merged) + + async def pin_lora(self, lora_id: int) -> bool: + """Prevent an adapter from being evicted.""" + loop = asyncio.get_event_loop() + results = await loop.run_in_executor( + self._executor, + self.engine.collective_rpc, + "pin_lora", + None, + (), + {"adapter_id": lora_id}, + None, + ) + return all(results) if isinstance(results, list) else results diff --git a/vllm_omni/entrypoints/omni.py b/vllm_omni/entrypoints/omni.py index e76fe834a..90b344c6f 100644 --- a/vllm_omni/entrypoints/omni.py +++ b/vllm_omni/entrypoints/omni.py @@ -221,6 +221,26 @@ class OmniBase: self.config_path = stage_configs_path self.stage_configs = load_stage_configs_from_yaml(stage_configs_path, base_engine_args=base_engine_args) + # Inject diffusion LoRA-related knobs from kwargs if not present in the stage config. + for cfg in self.stage_configs: + try: + if getattr(cfg, "stage_type", None) != "diffusion": + continue + if not hasattr(cfg, "engine_args") or cfg.engine_args is None: + cfg.engine_args = OmegaConf.create({}) + if kwargs.get("lora_path") is not None: + if not hasattr(cfg.engine_args, "lora_path") or cfg.engine_args.lora_path is None: + cfg.engine_args.lora_path = kwargs["lora_path"] + lora_scale = kwargs.get("lora_scale") + if lora_scale is None: + # Backwards compatibility for older callers. + lora_scale = kwargs.get("static_lora_scale") + if lora_scale is not None: + if not hasattr(cfg.engine_args, "lora_scale") or cfg.engine_args.lora_scale is None: + cfg.engine_args.lora_scale = lora_scale + except Exception as e: + logger.warning("Failed to inject LoRA config for stage: %s", e) + # Initialize connectors self.omni_transfer_config, self.connectors = initialize_orchestrator_connectors( self.config_path, worker_backend=worker_backend, shm_threshold_bytes=shm_threshold_bytes diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 2369a1000..3c44da8b3 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -33,7 +33,14 @@ from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, ChatCompletionResponse, ErrorResponse, + ModelCard, + ModelList, + ModelPermission, ) + +# yapf conflicts with isort for this block +# yapf: disable +# yapf: enable from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses @@ -73,6 +80,8 @@ from vllm_omni.entrypoints.openai.protocol.images import ( ) from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.lora.request import LoRARequest +from vllm_omni.lora.utils import stable_lora_int_id logger = init_logger(__name__) @@ -100,6 +109,31 @@ def _remove_route_from_router(router_obj, path: str, methods: set[str] | None = _remove_route_from_router(router, "/v1/chat/completions", {"POST"}) +class _DiffusionServingModels: + """Minimal OpenAIServingModels implementation for diffusion-only servers. + + vLLM's /v1/models route expects `app.state.openai_serving_models` to expose + `show_available_models()`. In pure diffusion mode we don't initialize the + full OpenAIServingModels (it depends on LLM-specific processors), so we + provide a lightweight fallback. + """ + + def __init__(self, base_model_paths: list[BaseModelPath]) -> None: + self._base_model_paths = base_model_paths + + async def show_available_models(self) -> ModelList: + return ModelList( + data=[ + ModelCard( + id=base_model.name, + root=base_model.model_path, + permission=[ModelPermission()], + ) + for base_model in self._base_model_paths + ] + ) + + # Server entry points @@ -330,6 +364,7 @@ async def omni_init_app_state( model_name = served_model_names[0] if served_model_names else args.model state.vllm_config = None state.diffusion_engine = engine_client + state.openai_serving_models = _DiffusionServingModels(base_model_paths) # Use for_diffusion method to create chat handler state.openai_serving_chat = OmniOpenAIServingChat.for_diffusion( @@ -823,6 +858,40 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request) "num_outputs_per_prompt": request.n, } + # Parse per-request LoRA (compatible with chat's extra_body.lora shape). + if request.lora is not None: + if not isinstance(request.lora, dict): + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="Invalid lora field: expected an object.", + ) + lora_body = request.lora + lora_name = lora_body.get("name") or lora_body.get("lora_name") or lora_body.get("adapter") + lora_path = ( + lora_body.get("local_path") + or lora_body.get("path") + or lora_body.get("lora_path") + or lora_body.get("lora_local_path") + ) + lora_scale = lora_body.get("scale") + if lora_scale is None: + lora_scale = lora_body.get("lora_scale") + lora_int_id = lora_body.get("int_id") + if lora_int_id is None: + lora_int_id = lora_body.get("lora_int_id") + if lora_int_id is None and lora_path: + lora_int_id = stable_lora_int_id(str(lora_path)) + + if not lora_name or not lora_path: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="Invalid lora object: both name and path are required.", + ) + + gen_params["lora_request"] = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)) + if lora_scale is not None: + gen_params["lora_scale"] = float(lora_scale) + # Parse and add size if provided if request.size: width, height = parse_size(request.size) diff --git a/vllm_omni/entrypoints/openai/protocol/images.py b/vllm_omni/entrypoints/openai/protocol/images.py index cb7c346ac..093ecaddf 100644 --- a/vllm_omni/entrypoints/openai/protocol/images.py +++ b/vllm_omni/entrypoints/openai/protocol/images.py @@ -8,6 +8,7 @@ for text-to-image generation, with vllm-omni specific extensions. """ from enum import Enum +from typing import Any from pydantic import BaseModel, Field, field_validator @@ -88,6 +89,18 @@ class ImageGenerationRequest(BaseModel): ) seed: int | None = Field(default=None, description="Random seed for reproducibility") + # vllm-omni extension for per-request LoRA. + # This mirrors the `extra_body.lora` convention in /v1/chat/completions. + lora: dict[str, Any] | None = Field( + default=None, + description=( + "Optional LoRA adapter for this request. Expected shape: " + "{name/path/scale/int_id}. Field names are flexible " + "(e.g. name|lora_name|adapter, path|lora_path|local_path, " + "scale|lora_scale, int_id|lora_int_id)." + ), + ) + # VAE memory optimizations (set at model init, included for completeness) vae_use_slicing: bool | None = Field(default=False, description="Enable VAE slicing") vae_use_tiling: bool | None = Field(default=False, description="Enable VAE tiling") diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 5d9f76d70..f35008260 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -64,7 +64,6 @@ from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls from vllm.entrypoints.utils import should_include_usage from vllm.inputs.data import PromptType, TokensPrompt from vllm.logger import init_logger -from vllm.lora.request import LoRARequest from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.tokenizers import TokenizerLike @@ -83,6 +82,8 @@ from vllm_omni.entrypoints.chat_utils import parse_chat_messages_futures from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin from vllm_omni.entrypoints.openai.protocol import OmniChatCompletionStreamResponse from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio +from vllm_omni.lora.request import LoRARequest +from vllm_omni.lora.utils import stable_lora_int_id from vllm_omni.outputs import OmniRequestOutput if TYPE_CHECKING: @@ -1883,6 +1884,7 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # Text-to-video parameters (ref: text_to_video.py) num_frames = extra_body.get("num_frames") guidance_scale_2 = extra_body.get("guidance_scale_2") # For video high-noise CFG + lora_body = extra_body.get("lora") logger.info( "Diffusion chat request %s: prompt=%r, ref_images=%d, params=%s", @@ -1926,6 +1928,33 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): if guidance_scale_2 is not None: gen_kwargs["guidance_scale_2"] = guidance_scale_2 + # Parse per-request LoRA (works for both AsyncOmniDiffusion and AsyncOmni). + if lora_body and isinstance(lora_body, dict): + try: + lora_name = lora_body.get("name") or lora_body.get("lora_name") or lora_body.get("adapter") + lora_path = ( + lora_body.get("local_path") + or lora_body.get("path") + or lora_body.get("lora_path") + or lora_body.get("lora_local_path") + ) + # using "or" directly here may be buggy if `scale=0` + lora_scale = lora_body.get("scale") + if lora_scale is None: + lora_scale = lora_body.get("lora_scale") + lora_int_id = lora_body.get("int_id") + if lora_int_id is None: + lora_int_id = lora_body.get("lora_int_id") + if lora_int_id is None and lora_path: + lora_int_id = stable_lora_int_id(str(lora_path)) + if lora_name and lora_path: + lora_req = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)) + gen_kwargs["lora_request"] = lora_req + if lora_scale is not None: + gen_kwargs["lora_scale"] = float(lora_scale) + except Exception as e: # pragma: no cover - safeguard + logger.warning("Failed to parse LoRA request: %s", e) + # Add reference image if provided if pil_images: if len(pil_images) == 1: diff --git a/vllm_omni/lora/__init__.py b/vllm_omni/lora/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/lora/request.py b/vllm_omni/lora/request.py new file mode 100644 index 000000000..55eb02ba4 --- /dev/null +++ b/vllm_omni/lora/request.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# for now, it suffices to use vLLM's implementation directly +# as this is a user-facing variable, defined here to so that user can directly import LoRARequest from vllm_omni +from vllm.lora.request import LoRARequest + +__all__ = ["LoRARequest"] diff --git a/vllm_omni/lora/utils.py b/vllm_omni/lora/utils.py new file mode 100644 index 000000000..9404d080f --- /dev/null +++ b/vllm_omni/lora/utils.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import hashlib + + +def stable_lora_int_id(lora_path: str) -> int: + """Return a deterministic positive integer ID for a LoRA adapter. + + vLLM uses `lora_int_id` as the adapter's cache key. Python's built-in + `hash()` is intentionally randomized per process (PYTHONHASHSEED), which + makes it unsuitable for persistent IDs. This helper derives a stable + 63-bit positive integer from the adapter path. + """ + digest = hashlib.sha256(lora_path.encode("utf-8")).digest() + value = int.from_bytes(digest[:8], byteorder="big", signed=False) & ((1 << 63) - 1) + return value or 1 + + +__all__ = ["stable_lora_int_id"] From 1fe8e71433875bcb62f7f8c0a5e5bb19f8d90768 Mon Sep 17 00:00:00 2001 From: XU Mingshi <91017482+mxuax@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:33:38 +0800 Subject: [PATCH 19/83] [Diffusion][Feature] Non-Intrusive Sequence Parallelism (SP) Support for Wan2.2 (#966) Signed-off-by: mxuax Signed-off-by: XU Mingshi <91017482+mxuax@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- .../diffusion/parallelism_acceleration.md | 2 +- .../text_to_video/text_to_video.py | 44 +++++++++++++++++++ .../models/wan2_2/wan2_2_transformer.py | 40 +++++++++++++++++ vllm_omni/diffusion/registry.py | 15 +++++-- 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/docs/user_guide/diffusion/parallelism_acceleration.md b/docs/user_guide/diffusion/parallelism_acceleration.md index 007976f1a..3dc9af454 100644 --- a/docs/user_guide/diffusion/parallelism_acceleration.md +++ b/docs/user_guide/diffusion/parallelism_acceleration.md @@ -48,7 +48,7 @@ The following table shows which models are currently supported by parallelism me | Model | Model Identifier | Ulysses-SP | Ring-SP | Tensor-Parallel | |-------|------------------|------------|---------|--------------------------| -| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ❌ | ❌ | ❌ | +| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ✅ | ✅ | ❌ | ### Tensor Parallelism diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 066714c54..c655fff14 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -3,11 +3,13 @@ import argparse import os +import time from pathlib import Path import numpy as np import torch +from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import detect_device_type, is_npu @@ -40,6 +42,23 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Enable CPU offloading for diffusion models.", ) + parser.add_argument( + "--ulysses_degree", + type=int, + default=1, + help="Number of GPUs used for ulysses sequence parallelism.", + ) + parser.add_argument( + "--ring_degree", + type=int, + default=1, + help="Number of GPUs used for ring sequence parallelism.", + ) + parser.add_argument( + "--enforce_eager", + action="store_true", + help="Disable torch.compile and force eager execution.", + ) return parser.parse_args() @@ -52,6 +71,13 @@ def main(): vae_use_slicing = is_npu() vae_use_tiling = is_npu() + # Configure parallel settings (only SP is supported for Wan) + # Note: cfg_parallel and tensor_parallel are not implemented for Wan models + parallel_config = DiffusionParallelConfig( + ulysses_degree=args.ulysses_degree, + ring_degree=args.ring_degree, + ) + # Check if profiling is requested via environment variable profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) @@ -62,12 +88,25 @@ def main(): boundary_ratio=args.boundary_ratio, flow_shift=args.flow_shift, enable_cpu_offload=args.enable_cpu_offload, + parallel_config=parallel_config, + enforce_eager=args.enforce_eager, ) if profiler_enabled: print("[Profiler] Starting profiling...") omni.start_profile() + # Print generation configuration + print(f"\n{'=' * 60}") + print("Generation Configuration:") + print(f" Model: {args.model}") + print(f" Inference steps: {args.num_inference_steps}") + print(f" Frames: {args.num_frames}") + print(f" Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}") + print(f" Video size: {args.width}x{args.height}") + print(f"{'=' * 60}\n") + + generation_start = time.perf_counter() frames = omni.generate( args.prompt, negative_prompt=args.negative_prompt, @@ -79,6 +118,11 @@ def main(): num_inference_steps=args.num_inference_steps, num_frames=args.num_frames, ) + generation_end = time.perf_counter() + generation_time = generation_end - generation_start + + # Print profiling results + print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)") # Extract video frames from OmniRequestOutput if isinstance(frames, list) and len(frames) > 0: diff --git a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py index f56833e48..deb626a97 100644 --- a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py +++ b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py @@ -17,6 +17,10 @@ from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinea from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.distributed.sp_plan import ( + SequenceParallelInput, + SequenceParallelOutput, +) logger = init_logger(__name__) @@ -497,6 +501,16 @@ class WanTransformer3DModel(nn.Module): This is an optimized version of the diffusers WanTransformer3DModel that uses vLLM's efficient QKVParallelLinear and RMSNorm implementations. + Sequence Parallelism: + This model supports non-intrusive SP via _sp_plan. The plan specifies: + - RoPE (cos/sin) splitting via rope module's split_output + - hidden_states splitting at first transformer block input + - Output gathering at proj_out layer + + The video sequence (flattened patches) is parallelized across GPUs. + + Note: Our "Sequence Parallelism" (SP) corresponds to "Context Parallelism" (CP) in diffusers. + Args: patch_size: 3D patch dimensions for video embedding (t_patch, h_patch, w_patch) num_attention_heads: Number of attention heads @@ -520,6 +534,29 @@ class WanTransformer3DModel(nn.Module): "to_qkv": ["to_q", "to_k", "to_v"], } + # Sequence Parallelism for Wan (following diffusers' _cp_plan pattern) + # + # The _sp_plan specifies sharding/gathering at module boundaries: + # - rope: Split both RoPE outputs (freqs_cos, freqs_sin) via split_output=True + # - blocks.0: Split hidden_states input at the first transformer block + # - proj_out: Gather outputs after the final projection layer + # + # Note: _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism) + _sp_plan = { + # Shard RoPE embeddings after rope module computes them + "rope": { + 0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True), # freqs_cos [1, seq, 1, dim] + 1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True), # freqs_sin [1, seq, 1, dim] + }, + # Shard hidden_states at first transformer block input + # (after patch_embedding + flatten + transpose) + "blocks.0": { + "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3), # [B, seq, dim] + }, + # Gather at proj_out (final linear projection before unpatchify) + "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3), + } + def __init__( self, patch_size: tuple[int, int, int] = (1, 2, 2), @@ -613,8 +650,11 @@ class WanTransformer3DModel(nn.Module): post_patch_height = height // p_h post_patch_width = width // p_w + # Compute RoPE embeddings (sharded by _sp_plan via split_output=True) rotary_emb = self.rope(hidden_states) + # Patch embedding and flatten to sequence + # (hidden_states is sharded at blocks.0 input by _sp_plan) hidden_states = self.patch_embedding(hidden_states) hidden_states = hidden_states.flatten(2).transpose(1, 2) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index 466a58e2c..d4429aff6 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -164,7 +164,10 @@ def _apply_sequence_parallel_if_enabled(model, od_config: OmniDiffusionConfig) - return # Find transformer model(s) in the pipeline that have _sp_plan - transformer_attrs = ["transformer", "dit", "unet"] + # Include transformer_2 for two-stage models (e.g., Wan MoE) + transformer_attrs = ["transformer", "transformer_2", "dit", "unet"] + applied_count = 0 + for attr in transformer_attrs: if not hasattr(model, attr): continue @@ -190,11 +193,17 @@ def _apply_sequence_parallel_if_enabled(model, od_config: OmniDiffusionConfig) - else ("ulysses" if sp_config.ulysses_degree > 1 else "ring") ) logger.info( - f"Applying sequence parallelism to {transformer.__class__.__name__} " + f"Applying sequence parallelism to {transformer.__class__.__name__} ({attr}) " f"(sp_size={sp_size}, mode={mode}, ulysses={sp_config.ulysses_degree}, ring={sp_config.ring_degree})" ) apply_sequence_parallel(transformer, sp_config, plan) - return # Only apply to first transformer found + applied_count += 1 + + if applied_count == 0: + logger.warning( + f"Sequence parallelism is enabled (sp_size={sp_size}) but no transformer with _sp_plan found. " + "SP hooks not applied. Consider adding _sp_plan to your transformer model." + ) except Exception as e: logger.warning(f"Failed to apply sequence parallelism: {e}. Continuing without SP hooks.") From 17e79336aa4abf230a186e6cbaa215a00dabf1d4 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Tue, 27 Jan 2026 12:14:19 +0800 Subject: [PATCH 20/83] [doc] add some additional information for the diffusion model support (#952) Signed-off-by: zjy0516 Signed-off-by: Jiangyun Zhu Co-authored-by: Hongsheng Liu --- .../model/adding_diffusion_model.md | 84 ++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/docs/contributing/model/adding_diffusion_model.md b/docs/contributing/model/adding_diffusion_model.md index 7eb56d5f5..e5257749a 100644 --- a/docs/contributing/model/adding_diffusion_model.md +++ b/docs/contributing/model/adding_diffusion_model.md @@ -44,7 +44,7 @@ vllm_omni/ # Step-by-step-implementation ## Step 1: Model Implementation -The diffusion pipeline’s implementation follows **HuggingFace Diffusers**, and components that do not need modification can be imported directly. +The diffusion pipeline’s implementation follows **HuggingFace Diffusers**. ### 1.1 Define the Pipeline Class Define the pipeline class, e.g., `QwenImageEditPipeline`, and initialize all required submodules, either from HuggingFace `diffusers` or custom implementations. In `QwenImageEditPipeline`, only `QwenImageTransformer2DModel` is re-implemented to support optimizations such as Ulysses-SP. When adding new models in the future, you can either reuse this re-implemented `QwenImageTransformer2DModel` or extend it as needed. @@ -73,10 +73,75 @@ def get_qwen_image_edit_post_process_func( The forward function of `QwenImageEditPipeline` follows the HuggingFace `diffusers` design for the most part. The key differences are: + As described in the overview, arguments are passed through `OnniDiffusionRequest`, so we need to get user parameters from it accordingly. ```python -prompt = req.prompt if req.prompt is not None else prompt +prompt = req.prompt ``` + pre/post-processing are handled by the framework elsewhere, so skip them. +### 1.4 Replace some ops or layers in DiT component + +vLLM-Omni provides a set of optimized operators with better performance and built-in support for parallelism, including attention, rotary embeddings (RoPE), and linear layers. + +Below is an example showing how to replace standard Transformer attention and FFN layers with vLLM-Omni implementations: + +```python +from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.layers.rope import RotaryEmbedding + +class MyAttention(nn.Module): + def __init__(self): + super().__init__() + self.attn = Attention() + self.to_qkv = QKVParallelLinear() + self.to_out = RowParallelLinear() + self.rope = RotaryEmbedding(is_neox_style=False) + + def forward(self, hidden_states): + qkv, _ = self.to_qkv(hidden_states) + q, k, v = qkv.split(...) + q, k = self.rope(...) + attn_output = self.attn(q, k, v) + output = self.to_out(attn_output) + +class MyFFN(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = ColumnParallelLinear() + self.fc2 = RowParallelLinear() + self.act = F.gelu + + def forward(self, hidden_states): + hidden, _ = self.fc1(hidden_states) + hidden = self.act(hidden) + output = self.fc2(hidden) + return output +``` + +In this example: + ++ Attention uses vLLM-Omni’s optimized attention kernel together with parallel QKV projection and RoPE. + ++ Linear layers are replaced with column- and row-parallel variants to enable tensor parallelism. + ++ The FFN follows a standard two-layer structure and can be further optimized (e.g., using fused or merged projections) if needed. + + +### 1.5 Provide a `_repeated_blocks` in DiT model +`_repeated_blocks` is the small and frequently-repeated block(s) of a model -- typically a transformer layer. + +It's used for torch compile optimizations. +```python +_repeated_blocks = ["QwenImageTransformerBlock"] +``` + + +### 1.6 (Optional) implement sequence parallelism +vLLM-Omni has a non-intrusive `_sp_plan` that enable sequence parallel without modifying `forward()` logic. +You can refer to [How to parallelize a new model](../../user_guide/diffusion/parallelism_acceleration.md) + + +### 1.7 (Optional) integrate with Cache-Dit +vLLM-Omni supports acceleration via [Cache-Dit](../../user_guide/diffusion/cache_dit_acceleration.md). Most models compatible with Diffusers can use Cache-Dit seamlessly. For new models, you can extend support by modifying`cache_dit_backend.py` + ## Step 2: Extend OmniDiffusionRequest Fields User-provided inputs are ultimately passed to the model’s forward method through OmniDiffusionRequest, so we add the required fields here to support the new model. ```python @@ -139,6 +204,21 @@ Key point for writing the example: + Save or display the generated results so users can validate the integration. +## Step 5: Open a Pull Request + +When submitting a pull request to add support for a new model, please include the following information in the PR description: + ++ Output verification: provide generation outputs to verify correctness and model behavior. + ++ Inference speed: provide a comparison with the corresponding implementation in Diffusers. + ++ Parallelism support: specify the supported parallel sizes and any relevant limitations. + ++ Cache acceleration: check whether the model can be accelerated using Cache-Dit or not. + + +Providing these details helps reviewers evaluate correctness, performance improvements, and parallel scalability of the new model integration. + # Testing For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md). From 4a3a1b6de8bae51973468545055b7e84be151c68 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Tue, 27 Jan 2026 12:15:35 +0800 Subject: [PATCH 21/83] [Perf] avoid cpu op in QwenImageCrossAttention (#942) Signed-off-by: zjy0516 --- .../diffusion/cache/teacache/extractors.py | 27 +++++++++- .../qwen_image/qwen_image_transformer.py | 49 ++++++++++--------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/vllm_omni/diffusion/cache/teacache/extractors.py b/vllm_omni/diffusion/cache/teacache/extractors.py index 01d453061..780297919 100644 --- a/vllm_omni/diffusion/cache/teacache/extractors.py +++ b/vllm_omni/diffusion/cache/teacache/extractors.py @@ -20,6 +20,8 @@ from typing import Any import torch import torch.nn as nn +from vllm_omni.diffusion.forward_context import get_forward_context + @dataclass class CacheContext: @@ -217,14 +219,37 @@ def extract_qwen_context( """Execute all Qwen transformer blocks.""" h = hidden_states e = encoder_hidden_states + encoder_mask = encoder_hidden_states_mask + hidden_states_mask = None # default + if module.parallel_config is not None and module.parallel_config.sequence_parallel_size > 1: + ctx = get_forward_context() + if ctx.sp_original_seq_len is not None and ctx.sp_padding_size > 0: + # Create mask for the full (padded) sequence + # valid positions = True, padding positions = False + batch_size = hidden_states.shape[0] + padded_seq_len = ctx.sp_original_seq_len + ctx.sp_padding_size + hidden_states_mask = torch.ones( + batch_size, + padded_seq_len, + dtype=torch.bool, + device=hidden_states.device, + ) + hidden_states_mask[:, ctx.sp_original_seq_len :] = False + + # if mask is all true, set it to None + if hidden_states_mask is not None and hidden_states_mask.all(): + hidden_states_mask = None + if encoder_mask is not None and encoder_mask.all(): + encoder_mask = None for block in module.transformer_blocks: e, h = block( hidden_states=h, encoder_hidden_states=e, - encoder_hidden_states_mask=encoder_hidden_states_mask, + encoder_hidden_states_mask=encoder_mask, temb=temb, image_rotary_emb=image_rotary_emb, joint_attention_kwargs=attention_kwargs, + hidden_states_mask=hidden_states_mask, ) return (h, e) diff --git a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py index 04890fbb9..054a5cb36 100644 --- a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py +++ b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py @@ -482,30 +482,6 @@ class QwenImageCrossAttention(nn.Module): hidden_states_mask: torch.Tensor | None = None, encoder_hidden_states_mask: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: - # Check for SP auto_pad: create attention mask dynamically if padding was applied - # In Ulysses mode, attention is computed on the FULL sequence (after All-to-All) - if ( - self.parallel_config is not None - and self.parallel_config.sequence_parallel_size > 1 - and hidden_states_mask is None - ): - ctx = get_forward_context() - if ctx.sp_original_seq_len is not None and ctx.sp_padding_size > 0: - # Create mask for the full (padded) sequence - # valid positions = True, padding positions = False - batch_size = hidden_states.shape[0] - padded_seq_len = ctx.sp_original_seq_len + ctx.sp_padding_size - hidden_states_mask = torch.ones( - batch_size, padded_seq_len, dtype=torch.bool, device=hidden_states.device - ) - hidden_states_mask[:, ctx.sp_original_seq_len :] = False - - # if mask is all true, set it to None - if hidden_states_mask is not None and hidden_states_mask.all(): - hidden_states_mask = None - if encoder_hidden_states_mask is not None and encoder_hidden_states_mask.all(): - encoder_hidden_states_mask = None - img_qkv, _ = self.to_qkv(hidden_states) q_size = self.query_num_heads * self.head_dim kv_size = self.kv_num_heads * self.head_dim @@ -951,6 +927,30 @@ class QwenImageTransformer2DModel(CachedTransformer): else self.time_text_embed(timestep, guidance, hidden_states, additional_t_cond) ) + # Check for SP auto_pad: create attention mask dynamically if padding was applied + # In Ulysses mode, attention is computed on the FULL sequence (after All-to-All) + hidden_states_mask = None # default + if self.parallel_config is not None and self.parallel_config.sequence_parallel_size > 1: + ctx = get_forward_context() + if ctx.sp_original_seq_len is not None and ctx.sp_padding_size > 0: + # Create mask for the full (padded) sequence + # valid positions = True, padding positions = False + batch_size = hidden_states.shape[0] + padded_seq_len = ctx.sp_original_seq_len + ctx.sp_padding_size + hidden_states_mask = torch.ones( + batch_size, + padded_seq_len, + dtype=torch.bool, + device=hidden_states.device, + ) + hidden_states_mask[:, ctx.sp_original_seq_len :] = False + + # if mask is all true, set it to None + if hidden_states_mask is not None and hidden_states_mask.all(): + hidden_states_mask = None + if encoder_hidden_states_mask is not None and encoder_hidden_states_mask.all(): + encoder_hidden_states_mask = None + for index_block, block in enumerate(self.transformer_blocks): encoder_hidden_states, hidden_states = block( hidden_states=hidden_states, @@ -960,6 +960,7 @@ class QwenImageTransformer2DModel(CachedTransformer): image_rotary_emb=image_rotary_emb, joint_attention_kwargs=attention_kwargs, modulate_index=modulate_index, + hidden_states_mask=hidden_states_mask, ) if self.zero_cond_t: From 77ff875170aa8ff725f911e6648bb2a32bf30329 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Tue, 27 Jan 2026 01:37:21 -0500 Subject: [PATCH 22/83] Support Qwen3 tts online serving (#968) Signed-off-by: linyueqian --- examples/online_serving/qwen3_tts/README.md | 170 +++++++++++++ .../qwen3_tts/openai_speech_client.py | 235 ++++++++++++++++++ .../online_serving/qwen3_tts/run_server.sh | 40 +++ .../openai_api/test_serving_speech.py | 81 +++++- .../omni_connectors/utils/serialization.py | 6 +- .../entrypoints/openai/protocol/audio.py | 38 ++- .../entrypoints/openai/serving_speech.py | 219 +++++++++++++++- .../modeling_qwen3_tts_tokenizer_v2.py | 4 +- 8 files changed, 772 insertions(+), 21 deletions(-) create mode 100644 examples/online_serving/qwen3_tts/README.md create mode 100644 examples/online_serving/qwen3_tts/openai_speech_client.py create mode 100755 examples/online_serving/qwen3_tts/run_server.sh diff --git a/examples/online_serving/qwen3_tts/README.md b/examples/online_serving/qwen3_tts/README.md new file mode 100644 index 000000000..2cddd0fbf --- /dev/null +++ b/examples/online_serving/qwen3_tts/README.md @@ -0,0 +1,170 @@ +# Qwen3-TTS Online Serving + +This directory contains examples for running Qwen3-TTS models with vLLM-Omni's online serving API. + +## Supported Models + +| Model | Task Type | Description | +|-------|-----------|-------------| +| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control | +| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description | +| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio | + +## Quick Start + +### 1. Start the Server + +```bash +# CustomVoice model (default) +./run_server.sh + +# Or specify task type +./run_server.sh CustomVoice +./run_server.sh VoiceDesign +./run_server.sh Base +``` + +### 2. Run the Client + +```bash +# CustomVoice: Use predefined speaker +python openai_speech_client.py \ + --text "你好，我是通义千问" \ + --voice Vivian \ + --language Chinese + +# CustomVoice with style instruction +python openai_speech_client.py \ + --text "今天天气真好" \ + --voice Ryan \ + --instructions "用开心的语气说" + +# VoiceDesign: Describe the voice style +python openai_speech_client.py \ + --task-type VoiceDesign \ + --text "哥哥，你回来啦" \ + --instructions "体现撒娇稚嫩的萝莉女声，音调偏高" + +# Base: Voice cloning +python openai_speech_client.py \ + --task-type Base \ + --text "Hello, this is a cloned voice" \ + --ref-audio /path/to/reference.wav \ + --ref-text "Original transcript of the reference audio" +``` + +### 3. Using curl + +```bash +# Simple TTS request +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, how are you?", + "voice": "Vivian", + "language": "English" + }' --output output.wav + +# With style instruction +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "I am so excited!", + "voice": "Vivian", + "instructions": "Speak with great enthusiasm" + }' --output excited.wav +``` + +## API Reference + +### Endpoint + +``` +POST /v1/audio/speech +``` + +This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters. + +### Request Body + +```json +{ + "input": "Text to synthesize", + "voice": "Vivian", + "response_format": "wav", + "task_type": "CustomVoice", + "language": "Auto", + "instructions": "Optional style instructions", + "ref_audio": "URL or base64 for voice cloning", + "ref_text": "Reference audio transcript", + "x_vector_only_mode": false, + "max_new_tokens": 2048 +} +``` + +> **Note:** The `model` field is optional when serving a single model, as the server already knows which model is loaded. + +### Response + +Returns audio data in the requested format (default: WAV). + +## Parameters + +### Standard OpenAI Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | string | required | Text to synthesize | +| `voice` | string | "Vivian" | Speaker/voice name | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `speed` | float | 1.0 | Playback speed (0.25-4.0) | +| `model` | string | optional | Model name (optional when serving single model) | + +### Qwen3-TTS Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `task_type` | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base | +| `language` | string | "Auto" | Language: Auto, Chinese, English, Japanese, Korean | +| `instructions` | string | "" | Voice style/emotion instructions | +| `max_new_tokens` | int | 2048 | Maximum tokens to generate | + +### Voice Clone Parameters (Base task) + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `ref_audio` | string | Yes* | Reference audio (file path, URL, or base64) | +| `ref_text` | string | No | Transcript of reference audio (for ICL mode) | +| `x_vector_only_mode` | bool | false | Use speaker embedding only (no ICL) | + +## Python Usage + +```python +import httpx + +# Simple request +response = httpx.post( + "http://localhost:8000/v1/audio/speech", + json={ + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "input": "Hello world", + "voice": "Vivian", + }, + timeout=300.0, +) + +with open("output.wav", "wb") as f: + f.write(response.content) +``` + +## Limitations + +- **No streaming**: Audio is generated completely before being returned. Streaming will be supported after the pipeline is disaggregated (see RFC #938). +- **Single request**: Batch processing is not yet optimized for online serving. + +## Troubleshooting + +1. **Connection refused**: Make sure the server is running on the correct port +2. **Out of memory**: Reduce `--gpu-memory-utilization` in run_server.sh +3. **Unsupported speaker**: Check supported speakers via model documentation +4. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning diff --git a/examples/online_serving/qwen3_tts/openai_speech_client.py b/examples/online_serving/qwen3_tts/openai_speech_client.py new file mode 100644 index 000000000..272e84265 --- /dev/null +++ b/examples/online_serving/qwen3_tts/openai_speech_client.py @@ -0,0 +1,235 @@ +"""OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech endpoint. + +This script demonstrates how to use the OpenAI-compatible speech API +to generate audio from text using Qwen3-TTS models. + +Examples: + # CustomVoice task (predefined speaker) + python openai_speech_client.py --text "Hello, how are you?" --voice Vivian + + # CustomVoice with emotion instruction + python openai_speech_client.py --text "I'm so happy!" --voice Vivian \ + --instructions "Speak with excitement" + + # VoiceDesign task (voice from description) + python openai_speech_client.py --text "Hello world" \ + --task-type VoiceDesign \ + --instructions "A warm, friendly female voice" + + # Base task (voice cloning) + python openai_speech_client.py --text "Hello world" \ + --task-type Base \ + --ref-audio "https://example.com/reference.wav" \ + --ref-text "This is the reference transcript" +""" + +import argparse +import base64 +import os + +import httpx + +# Default server configuration +DEFAULT_API_BASE = "http://localhost:8000" +DEFAULT_API_KEY = "EMPTY" + + +def encode_audio_to_base64(audio_path: str) -> str: + """Encode a local audio file to base64 data URL.""" + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + + # Detect MIME type from extension + audio_path_lower = audio_path.lower() + if audio_path_lower.endswith(".wav"): + mime_type = "audio/wav" + elif audio_path_lower.endswith((".mp3", ".mpeg")): + mime_type = "audio/mpeg" + elif audio_path_lower.endswith(".flac"): + mime_type = "audio/flac" + elif audio_path_lower.endswith(".ogg"): + mime_type = "audio/ogg" + else: + mime_type = "audio/wav" # Default + + with open(audio_path, "rb") as f: + audio_bytes = f.read() + audio_b64 = base64.b64encode(audio_bytes).decode("utf-8") + return f"data:{mime_type};base64,{audio_b64}" + + +def run_tts_generation(args) -> None: + """Run TTS generation via OpenAI-compatible /v1/audio/speech API.""" + + # Build request payload + payload = { + "model": args.model, + "input": args.text, + "voice": args.voice, + "response_format": args.response_format, + } + + # Add optional parameters + if args.instructions: + payload["instructions"] = args.instructions + if args.task_type: + payload["task_type"] = args.task_type + if args.language: + payload["language"] = args.language + if args.max_new_tokens: + payload["max_new_tokens"] = args.max_new_tokens + + # Voice clone parameters (Base task) + if args.ref_audio: + if args.ref_audio.startswith(("http://", "https://")): + payload["ref_audio"] = args.ref_audio + else: + payload["ref_audio"] = encode_audio_to_base64(args.ref_audio) + if args.ref_text: + payload["ref_text"] = args.ref_text + if args.x_vector_only: + payload["x_vector_only_mode"] = True + + print(f"Task type: {args.task_type or 'CustomVoice'}") + print(f"Text: {args.text}") + print(f"Voice: {args.voice}") + print("Generating audio...") + + # Make the API call + api_url = f"{args.api_base}/v1/audio/speech" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {args.api_key}", + } + + with httpx.Client(timeout=300.0) as client: + response = client.post(api_url, json=payload, headers=headers) + + if response.status_code != 200: + print(f"Error: {response.status_code}") + print(response.text) + return + + # Save audio response + output_path = args.output or "tts_output.wav" + with open(output_path, "wb") as f: + f.write(response.content) + print(f"Audio saved to: {output_path}") + + +def parse_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description="OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + # Server configuration + parser.add_argument( + "--api-base", + type=str, + default=DEFAULT_API_BASE, + help=f"API base URL (default: {DEFAULT_API_BASE})", + ) + parser.add_argument( + "--api-key", + type=str, + default=DEFAULT_API_KEY, + help="API key (default: EMPTY)", + ) + parser.add_argument( + "--model", + "-m", + type=str, + default="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + help="Model name/path", + ) + + # Task configuration + parser.add_argument( + "--task-type", + "-t", + type=str, + default=None, + choices=["CustomVoice", "VoiceDesign", "Base"], + help="TTS task type (default: CustomVoice)", + ) + + # Input text + parser.add_argument( + "--text", + type=str, + required=True, + help="Text to synthesize", + ) + + # Voice/speaker + parser.add_argument( + "--voice", + type=str, + default="Vivian", + help="Speaker/voice name (default: Vivian). Options: Vivian, Ryan, etc.", + ) + parser.add_argument( + "--language", + type=str, + default=None, + help="Language: Auto, Chinese, English, etc.", + ) + parser.add_argument( + "--instructions", + type=str, + default=None, + help="Voice style/emotion instructions", + ) + + # Base (voice clone) parameters + parser.add_argument( + "--ref-audio", + type=str, + default=None, + help="Reference audio file path or URL for voice cloning (Base task)", + ) + parser.add_argument( + "--ref-text", + type=str, + default=None, + help="Reference audio transcript for voice cloning (Base task)", + ) + parser.add_argument( + "--x-vector-only", + action="store_true", + help="Use x-vector only mode for voice cloning (no ICL)", + ) + + # Generation parameters + parser.add_argument( + "--max-new-tokens", + type=int, + default=None, + help="Maximum new tokens to generate", + ) + + # Output + parser.add_argument( + "--response-format", + type=str, + default="wav", + choices=["wav", "mp3", "flac", "pcm", "aac", "opus"], + help="Audio output format (default: wav)", + ) + parser.add_argument( + "--output", + "-o", + type=str, + default=None, + help="Output audio file path (default: tts_output.wav)", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + run_tts_generation(args) diff --git a/examples/online_serving/qwen3_tts/run_server.sh b/examples/online_serving/qwen3_tts/run_server.sh new file mode 100755 index 000000000..203ed76e8 --- /dev/null +++ b/examples/online_serving/qwen3_tts/run_server.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Launch vLLM-Omni server for Qwen3-TTS models +# +# Usage: +# ./run_server.sh # Default: CustomVoice model +# ./run_server.sh CustomVoice # CustomVoice model +# ./run_server.sh VoiceDesign # VoiceDesign model +# ./run_server.sh Base # Base (voice clone) model + +set -e + +TASK_TYPE="${1:-CustomVoice}" + +case "$TASK_TYPE" in + CustomVoice) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" + ;; + VoiceDesign) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign" + ;; + Base) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-Base" + ;; + *) + echo "Unknown task type: $TASK_TYPE" + echo "Supported: CustomVoice, VoiceDesign, Base" + exit 1 + ;; +esac + +echo "Starting Qwen3-TTS server with model: $MODEL" + +vllm-omni serve "$MODEL" \ + --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \ + --host 0.0.0.0 \ + --port 8000 \ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --enforce-eager \ + --omni diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index e2d33f317..82b5a4fef 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -10,8 +10,10 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin -from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio -from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.entrypoints.openai.protocol.audio import CreateAudio, OpenAICreateSpeechRequest +from vllm_omni.entrypoints.openai.serving_speech import ( + OmniOpenAIServingSpeech, +) from vllm_omni.outputs import OmniRequestOutput logger = logging.getLogger(__name__) @@ -265,3 +267,78 @@ class TestSpeechAPI: audio_obj = call_args[0] assert isinstance(audio_obj, CreateAudio) assert audio_obj.speed == 2.5 + + +class TestTTSMethods: + """Unit tests for TTS validation and parameter building.""" + + @pytest.fixture + def speech_server(self): + mock_engine_client = MagicMock() + mock_engine_client.errored = False + mock_engine_client.stage_list = None + mock_models = MagicMock() + mock_models.is_base_model.return_value = True + return OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=MagicMock(), + ) + + def test_is_tts_model(self, speech_server): + """Test TTS model detection.""" + # No stage_list -> False + assert speech_server._is_tts_model() is False + + # With qwen3_tts stage -> True + mock_stage = MagicMock() + mock_stage.model_stage = "qwen3_tts" + speech_server.engine_client.stage_list = [mock_stage] + assert speech_server._is_tts_model() is True + + def test_build_tts_prompt(self, speech_server): + """Test TTS prompt format.""" + prompt = speech_server._build_tts_prompt("Hello") + assert prompt == "<|im_start|>assistant\nHello<|im_end|>\n<|im_start|>assistant\n" + + def test_validate_tts_request_basic(self, speech_server): + """Test basic validation cases.""" + # Empty input + req = OpenAICreateSpeechRequest(input="") + assert speech_server._validate_tts_request(req) == "Input text cannot be empty" + + # Invalid language + req = OpenAICreateSpeechRequest(input="Hello", language="French") + assert "Invalid language" in speech_server._validate_tts_request(req) + + # Invalid speaker + req = OpenAICreateSpeechRequest(input="Hello", voice="Invalid") + assert "Invalid speaker" in speech_server._validate_tts_request(req) + + # Valid request + req = OpenAICreateSpeechRequest(input="Hello", voice="Vivian") + assert speech_server._validate_tts_request(req) is None + + def test_validate_tts_request_task_types(self, speech_server): + """Test task-specific validation.""" + # Base task requires ref_audio + req = OpenAICreateSpeechRequest(input="Hello", task_type="Base") + assert "ref_audio" in speech_server._validate_tts_request(req) + + # VoiceDesign requires instructions + req = OpenAICreateSpeechRequest(input="Hello", task_type="VoiceDesign") + assert "instructions" in speech_server._validate_tts_request(req) + + # ref_text only for Base + req = OpenAICreateSpeechRequest(input="Hello", ref_text="text") + assert "Base task" in speech_server._validate_tts_request(req) + + def test_build_tts_params(self, speech_server): + """Test TTS parameter building.""" + req = OpenAICreateSpeechRequest(input="Hello", voice="Ryan", language="English") + params = speech_server._build_tts_params(req) + + assert params["text"] == ["Hello"] + assert params["speaker"] == ["Ryan"] + assert params["language"] == ["English"] + assert params["task_type"] == ["CustomVoice"] diff --git a/vllm_omni/distributed/omni_connectors/utils/serialization.py b/vllm_omni/distributed/omni_connectors/utils/serialization.py index ee7ddf45f..74aa6d4d0 100644 --- a/vllm_omni/distributed/omni_connectors/utils/serialization.py +++ b/vllm_omni/distributed/omni_connectors/utils/serialization.py @@ -82,7 +82,11 @@ class OmniMsgpackEncoder: def _encode_tensor(self, tensor: torch.Tensor) -> dict[str, Any]: """Encode torch.Tensor to dict.""" - t = tensor.detach().contiguous().cpu().view(torch.uint8) + t = tensor.detach().contiguous().cpu() + # Handle 0-dimensional (scalar) tensors by reshaping to 1D first + if t.dim() == 0: + t = t.reshape(1) + t = t.view(torch.uint8) return { _TENSOR_MARKER: True, "dtype": str(tensor.dtype).removeprefix("torch."), diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 267d06291..d23460626 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -7,9 +7,15 @@ from pydantic import BaseModel, Field, field_validator class OpenAICreateSpeechRequest(BaseModel): input: str model: str | None = None - voice: Literal["alloy", "ash", "ballad", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "verse"] - instructions: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] + voice: str | None = Field( + default=None, + description="Voice to use. For OpenAI: alloy, echo, etc. For Qwen3-TTS: Vivian, Ryan, etc.", + ) + instructions: str | None = Field( + default=None, + description="Instructions for voice style/emotion (maps to 'instruct' for Qwen3-TTS)", + ) + response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" speed: float | None = Field( default=1.0, ge=0.25, @@ -17,6 +23,32 @@ class OpenAICreateSpeechRequest(BaseModel): ) stream_format: Literal["sse", "audio"] | None = "audio" + # Qwen3-TTS specific parameters + task_type: Literal["CustomVoice", "VoiceDesign", "Base"] | None = Field( + default=None, + description="TTS task type: CustomVoice, VoiceDesign, or Base (voice clone)", + ) + language: str | None = Field( + default=None, + description="Language code (e.g., 'Chinese', 'English', 'Auto')", + ) + ref_audio: str | None = Field( + default=None, + description="Reference audio for voice cloning (Base task). URL, base64, or file path.", + ) + ref_text: str | None = Field( + default=None, + description="Transcript of reference audio for voice cloning (Base task)", + ) + x_vector_only_mode: bool | None = Field( + default=None, + description="Use speaker embedding only without in-context learning (Base task)", + ) + max_new_tokens: int | None = Field( + default=None, + description="Maximum tokens to generate", + ) + @field_validator("stream_format") @classmethod def validate_stream_format(cls, v: str) -> str: diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index c6b87810e..d5bc20cde 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -1,4 +1,5 @@ import asyncio +from typing import Any from fastapi import Request from fastapi.responses import Response @@ -7,13 +8,149 @@ from vllm.logger import init_logger from vllm.utils import random_uuid from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin -from vllm_omni.entrypoints.openai.protocol.audio import AudioResponse, CreateAudio, OpenAICreateSpeechRequest +from vllm_omni.entrypoints.openai.protocol.audio import ( + AudioResponse, + CreateAudio, + OpenAICreateSpeechRequest, +) from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) +# TTS Configuration (currently supports Qwen3-TTS) +_TTS_MODEL_STAGES: set[str] = {"qwen3_tts"} +_TTS_SPEAKERS: set[str] = { + "Vivian", + "Serena", + "Uncle_Fu", + "Dylan", + "Eric", + "Ryan", + "Aiden", + "One_Anna", + "Sohee", +} +_TTS_LANGUAGES: set[str] = {"Auto", "Chinese", "English", "Japanese", "Korean"} +_TTS_MAX_INSTRUCTIONS_LENGTH = 500 +_TTS_MAX_NEW_TOKENS_MIN = 1 +_TTS_MAX_NEW_TOKENS_MAX = 4096 + class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): + def _is_tts_model(self) -> bool: + """Check if the current model is a supported TTS model.""" + stage_list = getattr(self.engine_client, "stage_list", None) + if stage_list: + for stage in stage_list: + model_stage = getattr(stage, "model_stage", None) + if model_stage in _TTS_MODEL_STAGES: + return True + return False + + def _validate_tts_request(self, request: OpenAICreateSpeechRequest) -> str | None: + """Validate TTS request parameters. Returns error message or None.""" + task_type = request.task_type or "CustomVoice" + + # Validate input is not empty + if not request.input or not request.input.strip(): + return "Input text cannot be empty" + + # Validate language + if request.language is not None and request.language not in _TTS_LANGUAGES: + return f"Invalid language '{request.language}'. Supported: {', '.join(sorted(_TTS_LANGUAGES))}" + + # Validate speaker for CustomVoice task + if task_type == "CustomVoice" and request.voice is not None: + if request.voice not in _TTS_SPEAKERS: + return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(_TTS_SPEAKERS))}" + + # Validate Base task requirements + if task_type == "Base": + if request.ref_audio is None: + return "Base task requires 'ref_audio' for voice cloning" + # Validate ref_audio format + if not (request.ref_audio.startswith(("http://", "https://")) or request.ref_audio.startswith("data:")): + return "ref_audio must be a URL (http/https) or base64 data URL (data:...)" + + # Validate cross-parameter dependencies + if task_type != "Base": + if request.ref_text is not None: + return "'ref_text' is only valid for Base task" + if request.x_vector_only_mode is not None: + return "'x_vector_only_mode' is only valid for Base task" + + # Validate VoiceDesign task requirements + if task_type == "VoiceDesign" and not request.instructions: + return "VoiceDesign task requires 'instructions' to describe the voice" + + # Validate instructions length + if request.instructions and len(request.instructions) > _TTS_MAX_INSTRUCTIONS_LENGTH: + return f"Instructions too long (max {_TTS_MAX_INSTRUCTIONS_LENGTH} characters)" + + # Validate max_new_tokens range + if request.max_new_tokens is not None: + if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN: + return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}" + if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX: + return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}" + + return None + + def _build_tts_prompt(self, text: str) -> str: + """Build TTS prompt from input text.""" + return f"<|im_start|>assistant\n{text}<|im_end|>\n<|im_start|>assistant\n" + + def _build_tts_params(self, request: OpenAICreateSpeechRequest) -> dict[str, Any]: + """Build TTS parameters from request. + + Processes each parameter if present, skips if not. + Values are wrapped in lists as required by the model. + """ + params: dict[str, Any] = {} + + # Text content (always required) + params["text"] = [request.input] + + # Task type + if request.task_type is not None: + params["task_type"] = [request.task_type] + else: + params["task_type"] = ["CustomVoice"] + + # Language + if request.language is not None: + params["language"] = [request.language] + else: + params["language"] = ["Auto"] + + # Speaker (voice) + if request.voice is not None: + params["speaker"] = [request.voice] + elif params["task_type"][0] == "CustomVoice": + params["speaker"] = ["Vivian"] # Default for CustomVoice + + # Instructions for style/emotion control + if request.instructions is not None: + params["instruct"] = [request.instructions] + else: + params["instruct"] = [""] + + # Voice clone parameters (used with Base task) + if request.ref_audio is not None: + params["ref_audio"] = [request.ref_audio] + if request.ref_text is not None: + params["ref_text"] = [request.ref_text] + if request.x_vector_only_mode is not None: + params["x_vector_only_mode"] = [request.x_vector_only_mode] + + # Generation parameters + if request.max_new_tokens is not None: + params["max_new_tokens"] = [request.max_new_tokens] + else: + params["max_new_tokens"] = [2048] + + return params + async def create_speech( self, request: OpenAICreateSpeechRequest, @@ -26,8 +163,16 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): for the API specification. This API mimics the OpenAI Create Speech API. - NOTE: This implementation does not currently support streaming audio - generation. The full audio is generated before the response is sent. + For Qwen3-TTS models, additional parameters are supported: + - task_type: "CustomVoice", "VoiceDesign", or "Base" + - language: Language code (e.g., "Chinese", "English", "Auto") + - voice: Speaker name (e.g., "Vivian", "Ryan") for CustomVoice + - instructions: Voice style/emotion instructions + - ref_audio: Reference audio for voice cloning (Base task) + - ref_text: Transcript of reference audio (Base task) + - x_vector_only_mode: Use speaker embedding only (Base task) + + NOTE: Streaming audio generation is not currently supported. """ error_check_ret = await self._check_model(request) @@ -41,11 +186,38 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): request_id = f"speech-{random_uuid()}" try: - prompt = {"prompt": request.input} + if self._is_tts_model(): + # Validate TTS parameters + validation_error = self._validate_tts_request(request) + if validation_error: + return self.create_error_response(validation_error) + + # Build TTS parameters and prompt + tts_params = self._build_tts_params(request) + prompt_text = self._build_tts_prompt(request.input) + prompt = { + "prompt": prompt_text, + "additional_information": tts_params, + } + else: + # Fallback for unsupported models + tts_params = {} + prompt = {"prompt": request.input} + + logger.info( + "TTS speech request %s: text=%r, task_type=%s", + request_id, + request.input[:50] + "..." if len(request.input) > 50 else request.input, + tts_params.get("task_type", ["unknown"])[0], + ) + sampling_params_list = self.engine_client.default_sampling_params_list generator = self.engine_client.generate( - prompt=prompt, request_id=request_id, sampling_params_list=sampling_params_list + prompt=prompt, + request_id=request_id, + sampling_params_list=sampling_params_list, + output_modalities=["audio"], ) final_output: OmniRequestOutput | None = None @@ -55,15 +227,36 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if final_output is None: return self.create_error_response("No output generated from the model.") - if final_output.final_output_type != "audio": - return self.create_error_response(f"Unexpected final output type: {final_output.final_output_type}") + # Extract audio from output + # Audio can be in final_output.multimodal_output or final_output.request_output.multimodal_output + audio_output = None + if hasattr(final_output, "multimodal_output") and final_output.multimodal_output: + audio_output = final_output.multimodal_output + if (not audio_output or "audio" not in audio_output) and hasattr(final_output, "request_output"): + if final_output.request_output and hasattr(final_output.request_output, "multimodal_output"): + audio_output = final_output.request_output.multimodal_output + + if not audio_output or "audio" not in audio_output: + return self.create_error_response("TTS model did not produce audio output.") + + audio_tensor = audio_output["audio"] + sample_rate = audio_output.get("sr", 24000) + if hasattr(sample_rate, "item"): + sample_rate = sample_rate.item() + + # Convert tensor to numpy + if hasattr(audio_tensor, "float"): + audio_tensor = audio_tensor.float().detach().cpu().numpy() + + # Squeeze batch dimension if present, but preserve channel dimension for stereo + if audio_tensor.ndim > 1: + audio_tensor = audio_tensor.squeeze() - audio_tensor = final_output.request_output.multimodal_output["audio"].float().detach().cpu().numpy() audio_obj = CreateAudio( audio_tensor=audio_tensor, - sample_rate=24000, - response_format=request.response_format, - speed=request.speed, + sample_rate=int(sample_rate), + response_format=request.response_format or "wav", + speed=request.speed or 1.0, stream_format=request.stream_format, base64_encode=False, ) @@ -74,7 +267,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): except asyncio.CancelledError: return self.create_error_response("Client disconnected") except ValueError as e: - # TODO: Use a vllm-specific Validation Error return self.create_error_response(str(e)) except Exception as e: - return self.create_error_response(f"{e} {e.__cause__}") + logger.exception("Speech generation failed: %s", e) + return self.create_error_response(f"Speech generation failed: {e}") diff --git a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py index 11c715f5f..31abe254b 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py +++ b/vllm_omni/model_executor/models/qwen3_tts/tokenizer_12hz/modeling_qwen3_tts_tokenizer_v2.py @@ -39,7 +39,6 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel from transformers.processing_utils import Unpack from transformers.utils import ModelOutput, auto_docstring, logging from transformers.utils.deprecation import deprecate_kwarg -from transformers.utils.generic import check_model_inputs from .configuration_qwen3_tts_tokenizer_v2 import ( Qwen3TTSTokenizerV2Config, @@ -495,7 +494,8 @@ class Qwen3TTSTokenizerV2DecoderTransformerModel(Qwen3TTSTokenizerV2DecoderPreTr # Initialize weights and apply final processing self.post_init() - @check_model_inputs + # Note: @check_model_inputs decorator removed for vLLM compatibility + # The decorator causes "unexpected keyword argument 'inputs_embeds'" error @auto_docstring def forward( self, From d12d3d711bed3573353091b49e9c2f9520d501a0 Mon Sep 17 00:00:00 2001 From: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:51:51 +0800 Subject: [PATCH 23/83] [test] fix test_image_generation_lora CI timeout (#975) Signed-off-by: AndyZhou952 --- .buildkite/pipeline.yml | 17 ----------------- .../test_images_generations_lora.py | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ca7ed7920..8f942b041 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,23 +41,6 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Diffusion Images API LoRA E2E" - timeout_in_minutes: 20 - depends_on: image-build - commands: - - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py - agents: - queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU - plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" - - label: "Diffusion Model CPU offloading Test" timeout_in_minutes: 20 depends_on: image-build diff --git a/tests/e2e/online_serving/test_images_generations_lora.py b/tests/e2e/online_serving/test_images_generations_lora.py index 38d3c9b89..e5fabcb81 100644 --- a/tests/e2e/online_serving/test_images_generations_lora.py +++ b/tests/e2e/online_serving/test_images_generations_lora.py @@ -84,7 +84,7 @@ class OmniServer: ) # Wait for server to be ready. - max_wait = 600 + max_wait = 1200 url = f"http://{self.host}:{self.port}/v1/models" start_time = time.time() while time.time() - start_time < max_wait: From 50b44303f196fa9bec4a5a505ef97104ae16d5ef Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Tue, 27 Jan 2026 16:37:25 +0800 Subject: [PATCH 24/83] [diffusion] add tp for FLUX.2-klein (#973) Signed-off-by: zjy0516 --- .../diffusion/parallelism_acceleration.md | 7 +- .../flux2_klein/flux2_klein_transformer.py | 81 ++++++++++++++----- 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/docs/user_guide/diffusion/parallelism_acceleration.md b/docs/user_guide/diffusion/parallelism_acceleration.md index 3dc9af454..09e4651ae 100644 --- a/docs/user_guide/diffusion/parallelism_acceleration.md +++ b/docs/user_guide/diffusion/parallelism_acceleration.md @@ -24,11 +24,12 @@ The following table shows which models are currently supported by parallelism me | **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ✅ | ✅ | ❌ | ❌ | | **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ❌ | ❌ | ❌ | | **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ | ✅ | -| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ | ✅ | ❌ | -| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ✅ | ✅ | ✅ | ❌ | -| **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ✅ | ✅ | ✅ | ❌ | +| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ | ✅ | ✅ | +| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ✅ | ✅ | ✅ | ✅ | +| **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ✅ | ✅ | ✅ | ✅ | | **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ✅ | ✅ | ❌ | ✅ (TP=2 only) | | **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ❌ | ❌ | ❌ | +| **FLUX.2-klein** | `black-forest-labs/FLUX.2-klein-4B` | ❌ | ❌ | ❌ | ✅ | !!! note "TP Limitations for Diffusion Models" diff --git a/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py b/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py index 312ac9b6a..ee10d2e0e 100644 --- a/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py +++ b/vllm_omni/diffusion/models/flux2_klein/flux2_klein_transformer.py @@ -29,7 +29,12 @@ from diffusers.models.embeddings import ( from diffusers.models.modeling_outputs import Transformer2DModelOutput from diffusers.models.normalization import AdaLayerNormContinuous from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata @@ -63,9 +68,20 @@ class Flux2FeedForward(nn.Module): inner_dim = int(dim * mult) dim_out = dim_out or dim - self.linear_in = nn.Linear(dim, inner_dim * 2, bias=bias) + self.linear_in = MergedColumnParallelLinear( + dim, + [inner_dim, inner_dim], + bias=bias, + return_bias=False, + ) self.act_fn = Flux2SwiGLU() - self.linear_out = nn.Linear(inner_dim, dim_out, bias=bias) + self.linear_out = RowParallelLinear( + inner_dim, + dim_out, + bias=bias, + input_is_parallel=True, + return_bias=False, + ) def forward(self, x: torch.Tensor) -> torch.Tensor: x = self.linear_in(x) @@ -101,15 +117,25 @@ class Flux2Attention(nn.Module): hidden_size=query_dim, head_size=self.head_dim, total_num_heads=self.heads, - disable_tp=True, bias=bias, ) + self.query_num_heads = self.to_qkv.num_heads + self.kv_num_heads = self.to_qkv.num_kv_heads self.norm_q = RMSNorm(dim_head, eps=eps) self.norm_k = RMSNorm(dim_head, eps=eps) self.to_out = nn.ModuleList( - [ReplicatedLinear(self.inner_dim, self.out_dim, bias=out_bias), nn.Dropout(dropout)] + [ + RowParallelLinear( + self.inner_dim, + self.out_dim, + bias=out_bias, + input_is_parallel=True, + return_bias=False, + ), + nn.Dropout(dropout), + ] ) if added_kv_proj_dim is not None: @@ -119,17 +145,25 @@ class Flux2Attention(nn.Module): hidden_size=added_kv_proj_dim, head_size=self.head_dim, total_num_heads=self.heads, - disable_tp=True, bias=added_proj_bias, ) - self.to_add_out = ReplicatedLinear(self.inner_dim, query_dim, bias=out_bias) + self.add_query_num_heads = self.add_kv_proj.num_heads + self.add_kv_num_heads = self.add_kv_proj.num_kv_heads + self.to_add_out = RowParallelLinear( + self.inner_dim, + query_dim, + bias=out_bias, + input_is_parallel=True, + return_bias=False, + ) self.rope = RotaryEmbedding(is_neox_style=False) self.attn = Attention( - num_heads=self.heads, + num_heads=self.query_num_heads, head_size=self.head_dim, softmax_scale=1.0 / (self.head_dim**0.5), causal=False, + num_kv_heads=self.kv_num_heads, ) def forward( @@ -148,17 +182,17 @@ class Flux2Attention(nn.Module): encoder_qkv, _ = self.add_kv_proj(encoder_hidden_states) encoder_query, encoder_key, encoder_value = encoder_qkv.chunk(3, dim=-1) - query = query.unflatten(-1, (self.heads, -1)) - key = key.unflatten(-1, (self.heads, -1)) - value = value.unflatten(-1, (self.heads, -1)) + query = query.unflatten(-1, (self.query_num_heads, -1)) + key = key.unflatten(-1, (self.kv_num_heads, -1)) + value = value.unflatten(-1, (self.kv_num_heads, -1)) query = self.norm_q(query) key = self.norm_k(key) if encoder_hidden_states is not None and self.added_kv_proj_dim is not None: - encoder_query = encoder_query.unflatten(-1, (self.heads, -1)) - encoder_key = encoder_key.unflatten(-1, (self.heads, -1)) - encoder_value = encoder_value.unflatten(-1, (self.heads, -1)) + encoder_query = encoder_query.unflatten(-1, (self.add_query_num_heads, -1)) + encoder_key = encoder_key.unflatten(-1, (self.add_kv_num_heads, -1)) + encoder_value = encoder_value.unflatten(-1, (self.add_kv_num_heads, -1)) encoder_query = self.norm_added_q(encoder_query) encoder_key = self.norm_added_k(encoder_key) @@ -189,9 +223,9 @@ class Flux2Attention(nn.Module): [context_len, hidden_states.shape[1] - context_len], dim=1, ) - encoder_hidden_states, _ = self.to_add_out(encoder_hidden_states) + encoder_hidden_states = self.to_add_out(encoder_hidden_states) - hidden_states, _ = self.to_out[0](hidden_states) + hidden_states = self.to_out[0](hidden_states) hidden_states = self.to_out[1](hidden_states) if encoder_hidden_states is not None: @@ -230,17 +264,23 @@ class Flux2ParallelSelfAttention(nn.Module): self.mlp_hidden_dim = int(query_dim * self.mlp_ratio) self.mlp_mult_factor = mlp_mult_factor - self.to_qkv_mlp_proj = nn.Linear( + self.to_qkv_mlp_proj = ColumnParallelLinear( self.query_dim, self.inner_dim * 3 + self.mlp_hidden_dim * self.mlp_mult_factor, bias=bias, + gather_output=True, ) self.mlp_act_fn = Flux2SwiGLU() self.norm_q = RMSNorm(dim_head, eps=eps) self.norm_k = RMSNorm(dim_head, eps=eps) - self.to_out = nn.Linear(self.inner_dim + self.mlp_hidden_dim, self.out_dim, bias=out_bias) + self.to_out = ColumnParallelLinear( + self.inner_dim + self.mlp_hidden_dim, + self.out_dim, + bias=out_bias, + gather_output=True, + ) self.rope = RotaryEmbedding(is_neox_style=False) self.attn = Attention( num_heads=self.heads, @@ -256,7 +296,7 @@ class Flux2ParallelSelfAttention(nn.Module): image_rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, **kwargs, ) -> torch.Tensor: - hidden_states = self.to_qkv_mlp_proj(hidden_states) + hidden_states, _ = self.to_qkv_mlp_proj(hidden_states) qkv, mlp_hidden_states = torch.split( hidden_states, [3 * self.inner_dim, self.mlp_hidden_dim * self.mlp_mult_factor], @@ -289,7 +329,8 @@ class Flux2ParallelSelfAttention(nn.Module): mlp_hidden_states = self.mlp_act_fn(mlp_hidden_states) hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=-1) - return self.to_out(hidden_states) + hidden_states, _ = self.to_out(hidden_states) + return hidden_states class Flux2SingleTransformerBlock(nn.Module): From 8efd912d3206ebdb6f7cac3cabf59eb126217881 Mon Sep 17 00:00:00 2001 From: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:39:07 +0800 Subject: [PATCH 25/83] [test] revert test flash attn file (#972) Signed-off-by: AndyZhou952 --- tests/diffusion/attention/test_flash_attn.py | 290 +++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 tests/diffusion/attention/test_flash_attn.py diff --git a/tests/diffusion/attention/test_flash_attn.py b/tests/diffusion/attention/test_flash_attn.py new file mode 100644 index 000000000..3f3862405 --- /dev/null +++ b/tests/diffusion/attention/test_flash_attn.py @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Test script for FlashAttention backend with padding handling. + +This script tests two main scenarios: +1. Case 1: Comparing padded vs unpadded inputs for batch_size=1 +2. Case 2: Comparing FlashAttention and SDPA backends for batch_size=2 with padding +""" + +import pytest +import torch + +from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata +from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionImpl +from vllm_omni.diffusion.attention.backends.sdpa import SDPAImpl + + +def create_attention_mask(batch_size: int, seq_len: int, valid_len: int, device: torch.device) -> torch.Tensor: + """ + Create attention mask where first valid_len tokens are valid (1) and rest are padding (0). + + Args: + batch_size: Batch size + seq_len: Total sequence length (including padding) + valid_len: Number of valid (non-padded) tokens + + Returns: + Attention mask of shape (batch_size, seq_len) + """ + mask = torch.zeros(batch_size, seq_len, dtype=torch.bool, device=device) + mask[:, :valid_len] = True + return mask + + +def pad_tensor(tensor: torch.Tensor, target_seq_len: int, pad_value: float = 0.0) -> torch.Tensor: + """ + Pad tensor along sequence dimension (dim=1). + + Args: + tensor: Input tensor of shape (batch_size, seq_len, num_heads, head_dim) + target_seq_len: Target sequence length after padding + pad_value: Value to use for padding + + Returns: + Padded tensor of shape (batch_size, target_seq_len, num_heads, head_dim) + """ + batch_size, seq_len, num_heads, head_dim = tensor.shape + if target_seq_len <= seq_len: + return tensor + + padding = torch.full( + (batch_size, target_seq_len - seq_len, num_heads, head_dim), pad_value, dtype=tensor.dtype, device=tensor.device + ) + return torch.cat([tensor, padding], dim=1) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="FlashAttention requires CUDA") +def test_padding_equivalence(): + """ + Case 1: Test that padded and unpadded inputs produce similar outputs. + + - Input A: batch_size=1, hidden_states (1, 48), encoder_hidden_states (1, 16) + Concatenated length: 64, NO attention_mask + - Input B: Same data but padded: hidden_states (1, 58), encoder_hidden_states (1, 26) + Concatenated length: 84, WITH attention_mask + + Expected: Output A and Output B should be very close. + """ + device = torch.device("cuda") + dtype = torch.bfloat16 + + # Configuration + batch_size = 1 + hidden_seq_len = 48 + encoder_seq_len = 16 + pad_length = 10 + num_heads = 8 + head_dim = 64 + + # Initialize FlashAttention + fa_impl = FlashAttentionImpl( + num_heads=num_heads, head_size=head_dim, softmax_scale=1.0 / (head_dim**0.5), causal=False + ) + + # Create base tensors with random values (same for both A and B) + torch.manual_seed(42) + hidden_states_base = torch.randn(batch_size, hidden_seq_len, num_heads, head_dim, device=device, dtype=dtype) + encoder_hidden_states_base = torch.randn( + batch_size, encoder_seq_len, num_heads, head_dim, device=device, dtype=dtype + ) + + # ========== Input A: Unpadded, no attention mask ========== + query_a = torch.cat([hidden_states_base, encoder_hidden_states_base], dim=1) + key_a = query_a.clone() + value_a = query_a.clone() + + attn_metadata_a = AttentionMetadata(attn_mask=None) + + output_a = fa_impl.forward(query=query_a, key=key_a, value=value_a, attn_metadata=attn_metadata_a) + + # ========== Input B: Padded with attention mask ========== + hidden_states_padded = pad_tensor(hidden_states_base, hidden_seq_len + pad_length) + encoder_hidden_states_padded = pad_tensor(encoder_hidden_states_base, encoder_seq_len + pad_length) + + query_b = torch.cat([hidden_states_padded, encoder_hidden_states_padded], dim=1) + key_b = query_b.clone() + value_b = query_b.clone() + + # Create attention mask + attn_mask_b = torch.cat( + [ + create_attention_mask(batch_size, hidden_seq_len + pad_length, hidden_seq_len, device), + create_attention_mask(batch_size, encoder_seq_len + pad_length, encoder_seq_len, device), + ], + dim=1, + ) + + attn_metadata_b = AttentionMetadata(attn_mask=attn_mask_b) + + output_b = fa_impl.forward(query=query_b, key=key_b, value=value_b, attn_metadata=attn_metadata_b) + + # Extract non-padded portion from output_b + output_b_unpadded = torch.cat( + [ + output_b[:, :hidden_seq_len, :, :], + output_b[:, hidden_seq_len + pad_length : hidden_seq_len + pad_length + encoder_seq_len, :, :], + ], + dim=1, + ) + + # Compare outputs + max_diff = torch.max(torch.abs(output_a - output_b_unpadded)).item() + mean_diff = torch.mean(torch.abs(output_a - output_b_unpadded)).item() + + print("\n=== Case 1: Padding Equivalence Test ===") + print(f"Output A shape: {output_a.shape}") + print(f"Output B shape: {output_b.shape}") + print(f"Output B unpadded shape: {output_b_unpadded.shape}") + print(f"Max absolute difference: {max_diff:.6f}") + print(f"Mean absolute difference: {mean_diff:.6f}") + + # Assert that outputs are close + # Using higher tolerance for bfloat16 + assert max_diff < 0.1, f"Max difference {max_diff} exceeds threshold 0.1" + assert mean_diff < 0.01, f"Mean difference {mean_diff} exceeds threshold 0.01" + + print("✓ Case 1 PASSED: Padded and unpadded outputs are very close!") + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="FlashAttention requires CUDA") +def test_fa_vs_sdpa(): + """ + Case 2: Compare FlashAttention and SDPA backends with padding. + + - batch_size=2 + - hidden_states: (2, 48) padded to (2, 58) + - encoder_hidden_states: (2, 16) padded to (2, 26) + - Concatenated length: 84 + - Compare FA and SDPA outputs + + Expected: FA and SDPA outputs should be very close. + """ + device = torch.device("cuda") + dtype = torch.bfloat16 + + # Configuration + batch_size = 2 + hidden_seq_len = 48 + encoder_seq_len = 16 + pad_length = 10 + num_heads = 8 + head_dim = 64 + + # Initialize both backends + fa_impl = FlashAttentionImpl( + num_heads=num_heads, head_size=head_dim, softmax_scale=1.0 / (head_dim**0.5), causal=False + ) + + sdpa_impl = SDPAImpl(num_heads=num_heads, head_size=head_dim, softmax_scale=1.0 / (head_dim**0.5), causal=False) + + # Create base tensors + torch.manual_seed(123) + hidden_states_base = torch.randn(batch_size, hidden_seq_len, num_heads, head_dim, device=device, dtype=dtype) + encoder_hidden_states_base = torch.randn( + batch_size, encoder_seq_len, num_heads, head_dim, device=device, dtype=dtype + ) + + # Pad tensors + hidden_states_padded = pad_tensor(hidden_states_base, hidden_seq_len + pad_length) + encoder_hidden_states_padded = pad_tensor(encoder_hidden_states_base, encoder_seq_len + pad_length) + + # Concatenate + query = torch.cat([hidden_states_padded, encoder_hidden_states_padded], dim=1) + key = query.clone() + value = query.clone() + + # Create attention mask + attn_mask = torch.cat( + [ + create_attention_mask(batch_size, hidden_seq_len + pad_length, hidden_seq_len, device), + create_attention_mask(batch_size, encoder_seq_len + pad_length, encoder_seq_len, device), + ], + dim=1, + ) + + attn_metadata = AttentionMetadata(attn_mask=attn_mask) + + # Run FlashAttention + output_fa = fa_impl.forward(query=query.clone(), key=key.clone(), value=value.clone(), attn_metadata=attn_metadata) + + # Run SDPA + # SDPA expects 4D attention mask: (batch_size, 1, seq_len, seq_len) or (batch_size, seq_len) + # For causal=False, we need to convert 2D mask to 4D + if attn_mask is not None: + # Expand mask for SDPA: (batch_size, seq_len) -> (batch_size, 1, 1, seq_len) + attn_mask_4d = attn_mask.unsqueeze(1).unsqueeze(2) + # Convert bool to float: True -> 0.0, False -> -inf + attn_mask_float = torch.zeros_like(attn_mask_4d, dtype=dtype) + attn_mask_float.masked_fill_(~attn_mask_4d, float("-inf")) + attn_metadata_sdpa = AttentionMetadata(attn_mask=attn_mask_float) + else: + attn_metadata_sdpa = AttentionMetadata(attn_mask=None) + + output_sdpa = sdpa_impl.forward( + query=query.clone(), key=key.clone(), value=value.clone(), attn_metadata=attn_metadata_sdpa + ) + + # Compare outputs (only compare valid regions) + output_fa_valid = torch.cat( + [ + output_fa[:, :hidden_seq_len, :, :], + output_fa[:, hidden_seq_len + pad_length : hidden_seq_len + pad_length + encoder_seq_len, :, :], + ], + dim=1, + ) + output_sdpa_valid = torch.cat( + [ + output_sdpa[:, :hidden_seq_len, :, :], + output_sdpa[:, hidden_seq_len + pad_length : hidden_seq_len + pad_length + encoder_seq_len, :, :], + ], + dim=1, + ) + + max_diff = torch.max(torch.abs(output_fa_valid - output_sdpa_valid)).item() + mean_diff = torch.mean(torch.abs(output_fa_valid - output_sdpa_valid)).item() + + print("\n=== Case 2: FA vs SDPA Comparison ===") + print(f"Batch size: {batch_size}") + print(f"FA output shape: {output_fa.shape}") + print(f"SDPA output shape: {output_sdpa.shape}") + print(f"Max absolute difference (valid region): {max_diff:.6f}") + print(f"Mean absolute difference (valid region): {mean_diff:.6f}") + + # Assert that outputs are close + # Using higher tolerance for bfloat16 and different implementations + assert max_diff < 0.01, f"Max difference {max_diff} exceeds threshold 0.01" + assert mean_diff < 0.001, f"Mean difference {mean_diff} exceeds threshold 0.001" + + print("✓ Case 2 PASSED: FA and SDPA outputs are very close!") + + +if __name__ == "__main__": + print("Running FlashAttention Padding Tests...") + print("=" * 60) + + # Try to run CUDA tests + if torch.cuda.is_available(): + try: + print("\n[Running Case 1: Padding Equivalence for FA]") + test_padding_equivalence() + except Exception as e: + print(f"✗ Case 1 failed: {e}") + import traceback + + traceback.print_exc() + + try: + print("\n[Running Case 2: FA vs SDPA]") + test_fa_vs_sdpa() + except Exception as e: + print(f"✗ Case 2 failed: {e}") + import traceback + + traceback.print_exc() + else: + raise RuntimeError("CUDA is not available") + print("\n" + "=" * 60) + print("Test suite completed!") From cadd77262d557750caab5718aa78dae94fdf3a2c Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Tue, 27 Jan 2026 16:45:32 +0800 Subject: [PATCH 26/83] [NPU] Upgrade to v0.14.0 (#820) Signed-off-by: gcanlin --- .../qwen3_omni/qwen3_omni_moe_talker.py | 28 +- .../stage_configs/npu/qwen3_omni_moe.yaml | 5 +- .../stage_configs/npu/qwen3_tts.yaml | 22 + vllm_omni/patch.py | 63 - vllm_omni/worker/npu/npu_ar_model_runner.py | 1235 +++++------------ vllm_omni/worker/npu/npu_ar_worker.py | 9 +- .../worker/npu/npu_generation_model_runner.py | 785 +++-------- vllm_omni/worker/npu/npu_generation_worker.py | 9 +- vllm_omni/worker/npu/npu_model_runner.py | 483 ++++++- 9 files changed, 994 insertions(+), 1645 deletions(-) create mode 100644 vllm_omni/model_executor/stage_configs/npu/qwen3_tts.yaml diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py index 2f1893e00..5b032d86e 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py @@ -563,7 +563,10 @@ class Qwen3OmniMoeTalkerSharedExpertWrapper(nn.Module): - mlp.shared_expert.{gate_proj, up_proj, down_proj}.weight - mlp.shared_expert_gate.weight (sibling, not child) - The wrapper applies: sigmoid(shared_expert_gate(x)) * shared_expert(x) + The wrapper applies: sigmoid(shared_expert_gate(x)) * shared_expert(x). + + It also exposes the underlying shared_expert interface to keep + compatibility with backends that split shared-expert computation. """ def __init__( @@ -575,9 +578,30 @@ class Qwen3OmniMoeTalkerSharedExpertWrapper(nn.Module): self._shared_expert = shared_expert self._shared_expert_gate = shared_expert_gate + @property + def gate_up_proj(self): + return self._shared_expert.gate_up_proj + + @property + def down_proj(self): + return self._shared_expert.down_proj + + @property + def act_fn(self): + return self._shared_expert.act_fn + + def expert_gate(self, x: torch.Tensor): + gate_out = self._shared_expert_gate(x) + if isinstance(gate_out, tuple): + return gate_out + return gate_out, None + def forward(self, x: torch.Tensor) -> torch.Tensor: out = self._shared_expert(x) - gate_values = F.sigmoid(self._shared_expert_gate(x)) # [batch, 1] + gate_out = self._shared_expert_gate(x) + if isinstance(gate_out, tuple): + gate_out = gate_out[0] + gate_values = F.sigmoid(gate_out) # [batch, 1] return gate_values * out # Broadcasting: [batch, 1] * [batch, hidden] diff --git a/vllm_omni/model_executor/stage_configs/npu/qwen3_omni_moe.yaml b/vllm_omni/model_executor/stage_configs/npu/qwen3_omni_moe.yaml index db51e6d7d..eadfea570 100644 --- a/vllm_omni/model_executor/stage_configs/npu/qwen3_omni_moe.yaml +++ b/vllm_omni/model_executor/stage_configs/npu/qwen3_omni_moe.yaml @@ -15,7 +15,7 @@ stage_args: worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 - enforce_eager: true + enforce_eager: false trust_remote_code: true engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" @@ -44,7 +44,7 @@ stage_args: worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.2 - enforce_eager: true + enforce_eager: false trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav # tensor_parallel_size: 2 @@ -75,6 +75,7 @@ stage_args: scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true + async_scheduling: false enable_prefix_caching: false engine_output_type: audio # Final output: audio waveform gpu_memory_utilization: 0.1 diff --git a/vllm_omni/model_executor/stage_configs/npu/qwen3_tts.yaml b/vllm_omni/model_executor/stage_configs/npu/qwen3_tts.yaml new file mode 100644 index 000000000..86b5ca60e --- /dev/null +++ b/vllm_omni/model_executor/stage_configs/npu/qwen3_tts.yaml @@ -0,0 +1,22 @@ +stage_args: + - stage_id: 0 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + devices: "0" + max_batch_size: 1 + engine_args: + model_stage: qwen3_tts + model_arch: Qwen3TTSForConditionalGeneration + worker_cls: vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker + scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler + enforce_eager: true + trust_remote_code: true + async_scheduling: false + enable_prefix_caching: false + engine_output_type: audio # Final output: audio waveform + gpu_memory_utilization: 0.1 + distributed_executor_backend: "mp" + max_num_batched_tokens: 1000000 + + final_output: true + final_output_type: audio diff --git a/vllm_omni/patch.py b/vllm_omni/patch.py index 9ecb5eeb0..687ff5186 100644 --- a/vllm_omni/patch.py +++ b/vllm_omni/patch.py @@ -14,7 +14,6 @@ from vllm_omni.engine import OmniEngineCoreOutput, OmniEngineCoreOutputs, OmniEn from vllm_omni.inputs.data import OmniTokensPrompt from vllm_omni.model_executor.layers.mrope import MRotaryEmbedding from vllm_omni.request import OmniRequest -from vllm_omni.utils import is_npu for module_name, module in sys.modules.items(): # only do patch on module of vllm, pass others @@ -32,65 +31,3 @@ for module_name, module in sys.modules.items(): module.Request = OmniRequest if hasattr(module, "EngineCoreRequest") and module.EngineCoreRequest == _OriginalEngineCoreRequest: module.EngineCoreRequest = OmniEngineCoreRequest - - -# Patch for vllm-ascend prefetch functions bug fix -# Issue: The original functions access forward_context attributes like -# prefetch_mlp_gate_up_proj, prefetch_mlp_down_proj, layer_idx without checking -# if they exist, which causes AttributeError when prefetch_mlp_enabled is not set. -# TODO: Remove this patch after upgrading to vllm-ascend v0.13.0 or later. -# This issue has been fixed in https://github.com/vllm-project/vllm-ascend/pull/5035 -if is_npu(): - import torch - import torch.nn as nn - from vllm.model_executor.models.qwen2_5_omni_thinker import Qwen2_5_VLImageInputs, Qwen2_5_VLVideoInputs - from vllm_ascend.ascend_forward_context import set_ascend_forward_context - - from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import ( - Qwen2_5OmniThinkerForConditionalGeneration, - ) - - class AscendQwen2_5OmniThinkerForConditionalGeneration(nn.Module): - def _process_image_input(self, image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]: - if image_input["type"] == "image_embeds": - return image_input["image_embeds"].type(self.visual.dtype) - - grid_thw = image_input["image_grid_thw"] - assert grid_thw.ndim == 2 - - pixel_values = image_input["pixel_values"].type(self.visual.dtype) - with set_ascend_forward_context(None, self.vllm_config): - image_embeds = self.visual(pixel_values, grid_thw=grid_thw) - # Split concatenated embeddings for each image item. - merge_size = self.visual.spatial_merge_size - sizes = grid_thw.prod(-1) // merge_size // merge_size - - return image_embeds.split(sizes.tolist()) - - def _process_video_input( - self, - video_input: Qwen2_5_VLVideoInputs, - video_hashes: list[str] | None = None, - cached_video_embeds: torch.Tensor | None = None, - ) -> torch.Tensor: - if video_input["type"] == "video_embeds": - return video_input["video_embeds"].type(self.visual.dtype) - - grid_thw = video_input["video_grid_thw"] - assert grid_thw.ndim == 2 - - pixel_values_videos = video_input["pixel_values_videos"].type(self.visual.dtype) - with set_ascend_forward_context(None, self.vllm_config): - video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw) - # Split concatenated embeddings for each video item. - merge_size = self.visual.spatial_merge_size - sizes = grid_thw.prod(-1) // merge_size // merge_size - - return video_embeds.split(sizes.tolist()) - - Qwen2_5OmniThinkerForConditionalGeneration._process_image_input = ( - AscendQwen2_5OmniThinkerForConditionalGeneration._process_image_input - ) - Qwen2_5OmniThinkerForConditionalGeneration._process_video_input = ( - AscendQwen2_5OmniThinkerForConditionalGeneration._process_video_input - ) diff --git a/vllm_omni/worker/npu/npu_ar_model_runner.py b/vllm_omni/worker/npu/npu_ar_model_runner.py index fcaf36795..0eedb61fb 100644 --- a/vllm_omni/worker/npu/npu_ar_model_runner.py +++ b/vllm_omni/worker/npu/npu_ar_model_runner.py @@ -3,48 +3,39 @@ from __future__ import annotations -import math +from copy import copy from typing import Any, NamedTuple import numpy as np import torch -import torch.nn as nn from vllm.config import CUDAGraphMode -from vllm.distributed import get_tensor_model_parallel_world_size, tensor_model_parallel_all_gather from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group -from vllm.distributed.parallel_state import get_pcp_group, get_pp_group, get_tp_group +from vllm.distributed.parallel_state import get_pp_group, get_tp_group from vllm.forward_context import get_forward_context from vllm.logger import logger from vllm.sequence import IntermediateTensors -from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput -from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec from vllm.v1.outputs import ( EMPTY_MODEL_RUNNER_OUTPUT, AsyncModelRunnerOutput, - ModelRunnerOutput, + ECConnectorOutput, make_empty_encoder_model_runner_output, ) from vllm.v1.spec_decode.metadata import SpecDecodeMetadata +from vllm.v1.structured_output.utils import apply_grammar_bitmask from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorOutput from vllm_ascend.ascend_forward_context import set_ascend_forward_context from vllm_ascend.attention.attention_v1 import AscendAttentionState -from vllm_ascend.attention.utils import AscendCommonAttentionMetadata # yapf conflicts with isort for this block # yapf: disable -from vllm_ascend.compilation.acl_graph import ( - update_attn_dcp_pcp_params, - update_attn_params, - update_mla_attn_dcp_pcp_params, - update_mla_attn_params, -) -from vllm_ascend.patch.worker.patch_module import patch_torch_npu_argsort -from vllm_ascend.spec_decode.interface import SpecDcodeType -from vllm_ascend.utils import ProfileExecuteDuration, enable_sp, lmhead_tp_enable +from vllm_ascend.compilation.acl_graph import update_full_graph_params +from vllm_ascend.ops.rotary_embedding import update_cos_sin +from vllm_ascend.utils import ProfileExecuteDuration +from vllm_omni.core.sched.omni_ar_scheduler import KVCacheTransferData from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.worker.npu.npu_model_runner import OmniNPUModelRunner @@ -62,6 +53,7 @@ class ExecuteModelState(NamedTuple): kv_connector_output: KVConnectorOutput | None attn_metadata: dict[str, Any] positions: torch.Tensor + ec_connector_output: ECConnectorOutput | None multimodal_outputs: Any class NPUARModelRunner(OmniNPUModelRunner): @@ -73,6 +65,7 @@ class NPUARModelRunner(OmniNPUModelRunner): # each model stage has their own hidden size self.hidden_size = self.model_config.hf_text_config.hidden_size self.inputs_embeds = self._make_buffer(self.max_num_tokens, self.hidden_size, dtype=self.dtype, numpy=False) + self.omni_connector = None def _make_buffer(self, *size, dtype, numpy=True): # Prevent ray from pinning the buffer due to large size @@ -87,640 +80,21 @@ class NPUARModelRunner(OmniNPUModelRunner): with maybe_disable_pin_memory_for_ray(self, total_bytes): return super()._make_buffer(*size, dtype=dtype, numpy=numpy) - def _prepare_inputs( - self, - scheduler_output: SchedulerOutput, - intermediate_tensors: IntermediateTensors | None = None, - ) -> tuple[ - dict[str, Any], - torch.Tensor, - np.ndarray, - int, - torch.Tensor, - int, - torch.Tensor, - SpecDecodeMetadata, - torch.Tensor | None, - torch.Tensor | None, - torch.Tensor | None, - int - ]: - total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens - assert total_num_scheduled_tokens > 0 - num_reqs = self.input_batch.num_reqs - assert num_reqs > 0 - - # OPTIMIZATION: Start copying the block table first. - # This way, we can overlap the copy with the following CPU operations. - self.input_batch.block_table.commit_block_table(num_reqs) - - # Get the number of scheduled tokens for each request. - req_ids = self.input_batch.req_ids - tokens = [scheduler_output.num_scheduled_tokens[i] for i in req_ids] - num_scheduled_tokens = np.array(tokens, dtype=np.int32) - - req_indices = np.repeat(self.arange_np[:num_reqs], - num_scheduled_tokens) - _, arange = self._get_cumsum_and_arange(num_scheduled_tokens) - positions_np = np.add( - self.input_batch.num_computed_tokens_cpu[req_indices], - arange, - ) - - self.input_batch.block_table.compute_slot_mapping( - req_indices, positions_np) - self.input_batch.block_table.commit_slot_mapping( - total_num_scheduled_tokens) - if self.pcp_size > 1: - if not self.vllm_config.model_config.use_mla: - self.generate_kv_idx(scheduler_output) - tokens, position_pcp, pcp_unpad_mask = self._update_tokens_for_pcp( - tokens) - num_scheduled_tokens = np.array(tokens, dtype=np.int32) - total_num_scheduled_tokens = sum(num_scheduled_tokens[:num_reqs]) - else: - position_pcp, pcp_unpad_mask = None, None - self.num_pcp_pads = self.num_pcp_pads[:num_reqs] - - total_num_pcp_pads = sum(self.num_pcp_pads) - max_num_scheduled_tokens = max(tokens) - num_valid_tokens = np.array([ - num_tokens - - len(scheduler_output.scheduled_spec_decode_tokens.get(i, [])) - for num_tokens, i in zip(tokens, req_ids) - ], - dtype=np.int32) - - if (self.use_aclgraph and total_num_scheduled_tokens - <= self.cudagraph_batch_sizes[-1]): - # Add padding to the batch size. - num_input_tokens = self.vllm_config.pad_for_cudagraph( - total_num_scheduled_tokens) - elif self.use_aclgraph and enable_sp(self.vllm_config): - # When using aclgraph, if total_num_scheduled_tokens exceeds the maximum graph size, - # the model will fall back to running its FX graph in eager mode. - # In this case, when sequence parallelism is enabled, we need to pad tokens to align - # with tp_size because pad_size cannot be captured by the FX graph - tp_size = self.vllm_config.parallel_config.tensor_parallel_size - num_input_tokens = math.ceil( - total_num_scheduled_tokens / tp_size) * tp_size - else: - # Eager mode. - num_input_tokens = total_num_scheduled_tokens - - # Get the attention state. - attn_state = self._build_attn_state(num_reqs, num_scheduled_tokens, - num_valid_tokens) - self.attn_state = attn_state # type: ignore - - # Determine if it's a splitfuse batch - with_prefill = attn_state not in [ - AscendAttentionState.DecodeOnly, AscendAttentionState.SpecDecoding - ] - - self.query_lens = torch.from_numpy(num_scheduled_tokens) - - # Get info across DP ranks. - # NOTE: maybe_padded_num_tokens is only used when using TorchAir with DP, - # Otherwise, it's just max_tokens_across_dp_cpu - (maybe_padded_num_tokens, num_tokens_across_dp, - with_prefill) = self._sync_metadata_across_dp(num_input_tokens, - with_prefill) - - # TODO: Now that num_input_tokens is basically identical with maybe_padded_num_tokens - # We should consider removing maybe_padded_num_tokens later - num_input_tokens = maybe_padded_num_tokens - - # Hot-Swap lora model - if self.lora_config: - self.set_active_loras(self.input_batch, num_scheduled_tokens) - - # Get request indices. - # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] - req_indices = np.repeat(self.arange_np[:num_reqs], - num_scheduled_tokens) - - # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] - # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - cu_num_tokens, arange = self._get_cumsum_and_arange( - num_scheduled_tokens) - - if self.pcp_size > 1: - positions_np = self.positions.np[:total_num_scheduled_tokens] - np.add(self.input_batch.num_computed_tokens_cpu[req_indices], - position_pcp[:total_num_scheduled_tokens], - out=positions_np) - else: - self.positions.np[:total_num_scheduled_tokens] = positions_np - - # Calculate M-RoPE positions. - # Only relevant for models using M-RoPE (e.g, Qwen2-VL) - if self.uses_mrope: - self._calc_mrope_positions(scheduler_output) - - # Only relevant for models using M-RoPE (e.g, Qwen2-VL) - self.mrope_positions.gpu[:, :total_num_scheduled_tokens].copy_( - self.mrope_positions.cpu[:, :total_num_scheduled_tokens], - non_blocking=True) - - # Get token indices. - # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2] - # where M is the max_model_len. - token_indices = (positions_np + - req_indices * self.input_batch.token_ids_cpu.shape[1]) - token_indices_tensor = torch.from_numpy(token_indices) - # Prepare input_ids. - # NOTE(woosuk): We use torch.index_select instead of np.take here - # because torch.index_select is much faster than np.take for large - # tensors. - torch.index_select(self.input_batch.token_ids_cpu_tensor.flatten(), - 0, - token_indices_tensor, - out=self.input_ids.cpu[:total_num_scheduled_tokens]) - if self.enable_prompt_embeds: - is_token_ids = self.input_batch.is_token_ids_tensor.flatten() - torch.index_select( - is_token_ids, - 0, - token_indices_tensor, - out=self.is_token_ids.cpu[:total_num_scheduled_tokens]) - - # Because we did not pre-allocate a massive prompt_embeds CPU tensor on - # the InputBatch, we need to fill in the prompt embeds into the expected - # spots in the GpuModelRunner's pre-allocated prompt_embeds tensor. - if self.input_batch.req_prompt_embeds and (self.is_multimodal_model or - self.enable_prompt_embeds): - output_idx = 0 - for req_idx in range(num_reqs): - num_sched = num_scheduled_tokens[req_idx] - - # Skip if this request doesn't have embeddings - if req_idx not in self.input_batch.req_prompt_embeds: - output_idx += num_sched - continue - - # Skip if no tokens scheduled - if num_sched <= 0: - output_idx += num_sched - continue - - req_embeds = self.input_batch.req_prompt_embeds[req_idx] - start_pos = self.input_batch.num_computed_tokens_cpu[req_idx] - - # Skip if trying to read beyond available embeddings - if start_pos >= req_embeds.shape[0]: - output_idx += num_sched - continue - - # Copy available embeddings - end_pos = start_pos + num_sched - actual_end = min(end_pos, req_embeds.shape[0]) - actual_num_sched = actual_end - start_pos - - if actual_num_sched > 0: - self.inputs_embeds.cpu[output_idx:output_idx + - actual_num_sched].copy_( - req_embeds[start_pos:actual_end] - ) - - output_idx += num_sched - - self.query_start_loc.np[0] = 0 - self.query_start_loc.np[1:num_reqs + 1] = cu_num_tokens - self.query_start_loc.copy_to_gpu() - - self.seq_lens.np[:num_reqs] = ( - self.input_batch.num_computed_tokens_cpu[:num_reqs] + - num_scheduled_tokens) - self.seq_lens.copy_to_gpu() - - # Fill unused with -1. Needed for reshape_and_cache - self.query_start_loc.gpu[num_reqs + 1:].fill_(-1) - self.seq_lens.gpu[num_reqs:].fill_(0) - - self.query_lens = torch.from_numpy(num_scheduled_tokens) - - # Copy the tensors to the NPU. - self._prepare_input_ids(scheduler_output, total_num_scheduled_tokens, - cu_num_tokens) - self.positions.cpu[total_num_scheduled_tokens:num_input_tokens].zero_() - self.positions.copy_to_gpu() - - attn_state = self._build_attn_state(num_reqs, num_scheduled_tokens, - num_valid_tokens) - self.attn_mask = self._make_attention_mask(attn_state) - self.attn_state = attn_state # type: ignore - - self.with_prefill = with_prefill - self.num_tokens_across_dp = num_tokens_across_dp - attn_metadata: dict[str, Any] = {} - - # Record the index of requests that should not be sampled, - # so that we could clear the sampled tokens before returning - num_tokens = [ - self.requests[r].num_tokens for r in self.input_batch.req_ids - ] - num_tokens_np = np.array(num_tokens, dtype=np.int32) - num_reqs = self.input_batch.num_reqs - if self.pcp_size > 1: - # while pcp > 1, we need the original num_scheduled_tokens before split - # to calculate discard_requests_mask - tokens_original = [ - scheduler_output.num_scheduled_tokens[i] for i in req_ids - ] - original_seq_lens_np = ( - self.input_batch.num_computed_tokens_cpu[:num_reqs] + - np.array(tokens_original, dtype=np.int32)) - discard_requests_mask = original_seq_lens_np < num_tokens_np - else: - discard_requests_mask = self.seq_lens.np[:num_reqs] < num_tokens_np - - discard_request_indices = np.nonzero(discard_requests_mask)[0] - self.num_discarded_requests = len(discard_request_indices) - self.discard_request_indices.np[:self.num_discarded_requests] = ( - discard_request_indices) - self.discard_request_indices.copy_to_gpu(self.num_discarded_requests) - - # _prepare_inputs may reorder the batch, so we must gather - # multi-modal outputs after that to ensure the correct order - if self.is_multimodal_model: - with self.maybe_get_ec_connector_output( - scheduler_output, - encoder_cache=self.encoder_cache, - ): - # Run the multimodal encoder if any. - self._execute_mm_encoder(scheduler_output) - - # NOTE(woosuk): To unify token ids and soft tokens (vision - # embeddings), we always use embeddings (rather than token ids) - # as input to the multimodal model, even when the input is text. - input_ids = self.input_ids.gpu[:total_num_scheduled_tokens] - mm_embeds, is_mm_embed = self._gather_mm_embeddings( - scheduler_output) - - inputs_embeds = self.model.embed_input_ids( - input_ids, - multimodal_embeddings=mm_embeds, - is_multimodal=is_mm_embed, - ) - - # TODO(woosuk): Avoid the copy. Optimize. - self.inputs_embeds.gpu[:total_num_scheduled_tokens].copy_( - inputs_embeds) - inputs_embeds = self.inputs_embeds.gpu[:num_input_tokens] - # -------------------------------------- Omni-new ------------------------------------------------- - input_ids = self.input_ids.gpu[:num_input_tokens] - # -------------------------------------- Omni-new ------------------------------------------------- - elif self.enable_prompt_embeds and get_pp_group().is_first_rank: - # Get the input embeddings for the tokens that are not input embeds, - # then put them into the appropriate positions. - # TODO(qthequartermasterman): Since even when prompt embeds are - # enabled, (a) not all requests will use prompt embeds, and (b) - # after the initial prompt is processed, the rest of the generated - # tokens will be token ids, it is not desirable to have the - # embedding layer outside of the acl graph all the time. The v0 - # engine avoids this by "double compiling" the acl graph, once - # with input_ids and again with inputs_embeds, for all num_tokens. - # If a batch only has token ids, then including the embedding layer - # in the acl graph will be more performant (like in the else case - # below). - token_ids_idx = self.is_token_ids.gpu[:total_num_scheduled_tokens] \ - .nonzero(as_tuple=False) \ - .squeeze(1) - # Some tokens ids may need to become embeds - if token_ids_idx.numel() > 0: - token_ids = self.input_ids.gpu[token_ids_idx] - tokens_to_embeds = self.model.embed_input_ids( - input_ids=token_ids) - self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds - - inputs_embeds = self.inputs_embeds.gpu[:num_input_tokens] - input_ids = None - else: - # For text-only models, we use token ids as input. - # While it is possible to use embeddings as input just like the - # multimodal models, it is not desirable for performance since - # then the embedding layer is not included in the ACL graph. - input_ids = self.input_ids.gpu[:num_input_tokens] - inputs_embeds = None - positions = self.positions.gpu[:num_input_tokens] - if self.uses_mrope: - positions = self.mrope_positions.gpu[:, :num_input_tokens] - - # -------------------------------------- Omni-new ------------------------------------------------- - self._omni_num_scheduled_tokens_np = num_scheduled_tokens - - # Note: only prefill need collect additional_information for now. - # Decode don't need per_req_additional_information anymore. - if inputs_embeds is not None: - # Prefill: overlay prompt_embeds and collect additional_information - self._collect_additional_information_for_prefill(num_scheduled_tokens) - - if hasattr(self.model, "has_preprocess") and self.model.has_preprocess: - # Overlay custom prompt_embeds per request for the prompt portion; - # collect additional_information (tensor/list) for prefill portion only - for req_index, req_id in enumerate(self.input_batch.req_ids): - req_state = self.requests.get(req_id) - req_infos = getattr(req_state, "additional_information_cpu", None) if req_state is not None else None - - start_offset = int(self.query_start_loc.cpu[req_index]) - sched_tokens = int(num_scheduled_tokens[req_index]) - s, e = start_offset, start_offset + sched_tokens - span_len = int(e) - int(s) - - # call the custom process function - req_input_ids, req_embeds, update_dict = self.model.preprocess( - input_ids=input_ids[s:e], input_embeds=inputs_embeds[s:e], **req_infos - ) - # TODO(Peiqi): the merge stage could move out from the critical path - self._merge_additional_information_update(req_id, update_dict) - - # update the inputs_embeds and input_ids - seg_len = min(span_len, req_embeds.shape[0]) - inputs_embeds[s : s + seg_len] = req_embeds[:seg_len] - if isinstance(req_input_ids, torch.Tensor) and req_input_ids.numel() == seg_len: - input_ids[s : s + seg_len] = req_input_ids - - # -------------------------------------- Omni-new ------------------------------------------------- - - - # type: ignore - if get_pp_group().is_first_rank: - intermediate_tensors = None - else: - assert intermediate_tensors is not None - assert self.intermediate_tensors is not None - # If both flashcomm1 and pp are used simultaneously, - # the shape of the received data and the shape of the space to be copied to will not match, - # requiring a recalculation of the incoming data's shape. - tp_size = get_tensor_model_parallel_world_size() - num_input_tokens_with_flashcomm1 = num_input_tokens - if enable_sp(): - num_input_tokens_with_flashcomm1 = (num_input_tokens + - tp_size - 1) // tp_size - for k, v in intermediate_tensors.items(): - self.intermediate_tensors[ - k][:num_input_tokens_with_flashcomm1].copy_( - v[:num_input_tokens_with_flashcomm1], - non_blocking=True) - intermediate_tensors = IntermediateTensors({ - k: - v[:num_input_tokens_with_flashcomm1] - for k, v in self.intermediate_tensors.items() - }) - - use_spec_decode = len( - scheduler_output.scheduled_spec_decode_tokens) > 0 - if not use_spec_decode: - # NOTE(woosuk): Due to chunked prefills, the batch may contain - # partial requests. While we should not sample any token - # from these partial requests, we do so for simplicity. - # We will ignore the sampled tokens from the partial requests. - # TODO: Support prompt logprobs. - spec_decode_metadata = None - if self.pcp_size * self.dcp_size > 1: - logits_indices = torch.from_numpy( - cu_num_tokens - ) * self.pcp_size - self.num_pcp_pads[:num_reqs] - 1 - logits_indices = logits_indices.pin_memory().to( - self.device, non_blocking=True) - else: - logits_indices = self.query_start_loc.gpu[1:num_reqs + 1] - 1 - else: - # Get the number of draft tokens for each request. - # Iterate over the dictionary rather than all requests since not all - # requests have draft tokens. - num_draft_tokens = np.zeros(num_reqs, dtype=np.int32) - # For chunked prefills, use -1 as mask rather than 0, as guided - # decoding may rollback speculative tokens. - num_decode_draft_tokens = np.full(num_reqs, -1, dtype=np.int32) - for req_id, draft_token_ids in ( - scheduler_output.scheduled_spec_decode_tokens.items()): - req_idx = self.input_batch.req_id_to_index[req_id] - num_draft_tokens[req_idx] = len(draft_token_ids) - num_decode_draft_tokens[req_idx] = (len(draft_token_ids) if ( - self.input_batch.num_computed_tokens_cpu[req_idx] - >= self.input_batch.num_prompt_tokens[req_idx]) else -1) - - spec_decode_metadata = self._calc_spec_decode_metadata( - num_draft_tokens, cu_num_tokens, self.num_pcp_pads[:num_reqs]) - logits_indices = spec_decode_metadata.logits_indices - - # For DECODE only cuda graph of some attention backends (e.g., GDN). - self.num_decode_draft_tokens.np[: - num_reqs] = num_decode_draft_tokens - self.num_decode_draft_tokens.np[num_reqs:].fill(-1) - self.num_decode_draft_tokens.copy_to_gpu() - # save logits_indices for pcp spec decode usage - self.logits_indices = logits_indices - - # Used in the below loop. - # query_start_loc_cpu = self.query_start_loc.cpu[:num_reqs + 1] - num_computed_tokens_cpu = ( - self.input_batch.num_computed_tokens_cpu_tensor[:num_reqs]) - self.spec_decode_common_attn_metadata = None - if use_spec_decode and self.need_accepted_tokens: - self.num_accepted_tokens.np[:num_reqs] = ( - self.input_batch.num_accepted_tokens_cpu[:num_reqs]) - self.num_accepted_tokens.np[num_reqs:].fill(1) - self.num_accepted_tokens.copy_to_gpu() - - if self.speculative_config and self.pcp_size > 1: - self._generate_pcp_mtp_input( - num_reqs, scheduler_output.total_num_scheduled_tokens, - scheduler_output.num_scheduled_tokens) - - long_seq_metadata = self._generate_pcp_metadata( - total_num_scheduled_tokens) - # Prepare the attention metadata for each KV cache group and make layers - # in the same group share the same metadata. - for kv_cache_group_id, kv_cache_group_spec in enumerate( - self.kv_cache_config.kv_cache_groups): - # NOTE: This is strange, why did we use total_num_scheduled_tokens before? - slot_mapping_size = (total_num_scheduled_tokens - if self.pcp_size == 1 else - total_num_scheduled_tokens * self.pcp_size - - total_num_pcp_pads) - if isinstance(kv_cache_group_spec.kv_cache_spec, - EncoderOnlyAttentionSpec): - # Encoder-only layers do not have KV cache, so we need to - # create a dummy block table and slot mapping for them. - blk_table_tensor = torch.zeros( - (num_reqs, 1), - dtype=torch.int32, - device=self.device, - ) - slot_mapping = torch.zeros( - (total_num_scheduled_tokens, ), - dtype=torch.int64, - device=self.device, - ) - else: - blk_table = self.input_batch.block_table[kv_cache_group_id] - blk_table_tensor = blk_table.get_device_tensor() - blk_table.slot_mapping.gpu[slot_mapping_size:].fill_(0) - if self.pcp_size > 1: - slot_mapping_for_pcp = blk_table.slot_mapping.gpu[: - long_seq_metadata - . - num_actual_tokens_pcp_padded] - slot_mapping_for_pcp[slot_mapping_size:].fill_(-1) - assert pcp_unpad_mask is not None - pcp_padded_slot_mapping = self.pcp_padded_slot_mapping[: - pcp_unpad_mask - . - shape[ - 0]] - pcp_padded_slot_mapping.fill_(-1) - pcp_padded_slot_mapping[ - pcp_unpad_mask] = slot_mapping_for_pcp[: - slot_mapping_size] - slot_mapping_for_pcp[:long_seq_metadata. - num_actual_tokens_pcp_padded] = pcp_padded_slot_mapping - blk_table.slot_mapping.gpu[:long_seq_metadata.num_actual_tokens_pcp_padded] = \ - slot_mapping_for_pcp - slot_mapping = blk_table.slot_mapping.gpu - - # NOTE: This is a temporary hack, now in GPUModelRunner, this prepare_inputs - # has been split to multiple parts, and there are 3 parts that is related to this - # `num_reqs`, we'll take `query_start_loc` as an example: - # 1. self.query_start_loc.np[1 : num_reqs + 1] = cu_num_tokens - # 2. get `num_reqs_padded`, this depends on dispatcher and which is why we have the - # following simplified `dispatch` logic here, we try to minimize the impact - # 3. query_start_loc = self.query_start_loc.gpu[: num_reqs_padded + 1] - uniform_decode = (max_num_scheduled_tokens == self.uniform_decode_query_len) \ - and (total_num_scheduled_tokens == max_num_scheduled_tokens * num_reqs) - - # TODO: We should make this official ASAP. Also note that if we pad here, - # the builders won’t need to add any extra padding. - if self.compilation_config.cudagraph_mode.decode_mode() == CUDAGraphMode.FULL and \ - uniform_decode: - num_reqs_padded = num_input_tokens // self.uniform_decode_query_len - pad_size = num_reqs_padded - num_reqs - if pad_size > 0: - last_query_loc = self.query_start_loc.gpu[num_reqs] - - steps = torch.arange(1, - pad_size + 1, - device=self.device, - dtype=self.query_start_loc.gpu.dtype) - fill_values = last_query_loc + ( - steps * self.uniform_decode_query_len) - - self.query_start_loc.gpu[num_reqs + 1:num_reqs_padded + - 1] = fill_values - # So we are trying to simulate the behavior of GPUModelRunner's - # prepare_inputs for uniform decode mode by padding query_start_loc - num_reqs = num_reqs_padded - - # Make AscendCommonAttentionMetadata - common_attn_metadata = AscendCommonAttentionMetadata( - query_start_loc=self.query_start_loc.gpu[:num_reqs + 1], - query_start_loc_cpu=self.query_start_loc.cpu[:num_reqs + 1], - seq_lens_cpu=self.seq_lens.cpu[:num_reqs], - seq_lens=self.seq_lens.gpu[:num_reqs], - num_reqs=num_reqs, - num_actual_tokens=slot_mapping_size, - num_input_tokens=num_input_tokens, - actual_seq_lengths_q=self.actual_seq_lengths_q, - # TODO: change this to the right block table for linear attn - block_table_tensor=blk_table_tensor[:num_reqs], - slot_mapping=slot_mapping, - num_computed_tokens_cpu=num_computed_tokens_cpu, - positions=self.positions.gpu, - attn_mask=self.attn_mask, - spec_attn_mask=self.spec_attn_mask, - attn_state=self.attn_state, - is_only_prefill=bool(np.all(num_valid_tokens != 1)), - max_query_len=max_num_scheduled_tokens, - decode_token_per_req=self.decode_token_per_req, - prefill_context_parallel_metadata=long_seq_metadata, - ) - - if self.speculative_config and self.pcp_size > 1: - # For pcp + spec decode, we flatten block_table - # to avoid irregular spec_attn_mask shape, e.g., - # num_decode_req=2, num_prefill_req=3, num_speculative_tokens=1, - # ori block_table: # [d0, d1, p0, p1, p2] - # (num_reqs_d + num_reqs_p, max_num_blocks), - # flattened block_table: [d0, d0, d1, d1, p0, p1, p2] - # (num_reqs_d * decode_threshold + num_reqs_p, max_num_blocks), - ori_query_lens = self.query_start_loc_pcp_full.cpu[1:num_reqs+1] - \ - self.query_start_loc_pcp_full.cpu[:num_reqs] - num_prefill_reqs = (ori_query_lens - > self.decode_threshold).sum().item() - num_decode_reqs = num_reqs - num_prefill_reqs - num_decode_reqs_flatten = num_decode_reqs * self.decode_threshold - blk_table_tensor[ - num_decode_reqs_flatten:num_decode_reqs_flatten + - num_prefill_reqs].copy_( - blk_table_tensor[num_decode_reqs:num_decode_reqs + - num_prefill_reqs].clone()) - blk_table_tensor[:num_decode_reqs_flatten].copy_( - blk_table_tensor[:num_decode_reqs].repeat_interleave( - self.decode_threshold, dim=0)) - common_attn_metadata.block_table_tensor = \ - blk_table_tensor[:num_decode_reqs_flatten + num_prefill_reqs] - - if self.speculative_config and \ - self.spec_decode_common_attn_metadata is None: - self.spec_decode_common_attn_metadata = common_attn_metadata - - for attn_group in self.attn_groups[kv_cache_group_id]: - common_prefix_len = 0 - extra_attn_metadata_args = {} - builder = attn_group.get_metadata_builder() - if isinstance(builder, GDNAttentionMetadataBuilder): - if use_spec_decode: - patch_torch_npu_argsort() - extra_attn_metadata_args = dict( - num_accepted_tokens=self.num_accepted_tokens. - gpu[:num_reqs], - num_decode_draft_tokens_cpu=self. - num_decode_draft_tokens.cpu[:num_reqs], - ) - attn_metadata_i = builder.build( - common_prefix_len=common_prefix_len, - common_attn_metadata=common_attn_metadata, - **extra_attn_metadata_args) - elif self.model_config.runner_type == "pooling": - attn_metadata_i = builder.build( - common_prefix_len=common_prefix_len, - common_attn_metadata=common_attn_metadata, - **extra_attn_metadata_args) - else: - attn_metadata_i = builder.build( - common_prefix_len=common_prefix_len, - common_attn_metadata=common_attn_metadata, - model=self.get_model(), - **extra_attn_metadata_args) - - for layer_name in attn_group.layer_names: - attn_metadata[layer_name] = attn_metadata_i - - if lmhead_tp_enable(): - max_num_reqs_across_dp = self.max_num_reqs * self.uniform_decode_query_len - logits_indices = nn.functional.pad( - logits_indices, - (0, max_num_reqs_across_dp - logits_indices.shape[0])) - - return (attn_metadata, positions, num_scheduled_tokens, - num_input_tokens, num_tokens_across_dp, - maybe_padded_num_tokens, logits_indices, spec_decode_metadata, - input_ids, inputs_embeds, intermediate_tensors, - max_num_scheduled_tokens) - @torch.inference_mode() def execute_model( self, scheduler_output: SchedulerOutput, intermediate_tensors: IntermediateTensors | None = None, - ) -> OmniModelRunnerOutput | IntermediateTensors | None: + ) -> OmniModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors | None: if self.execute_model_state is not None: raise RuntimeError("State error: sample_tokens() must be called " "after execute_model() returns None.") + # -------------------------------------- Omni-new ------------------------------------------------- + # [Omni] Handle KV transfer BEFORE updating states (which removes finished requests) + self.kv_extracted_req_ids = self._handle_finished_requests_kv_transfer(scheduler_output) + # -------------------------------------- Omni-new ------------------------------------------------- + with ProfileExecuteDuration().capture_async("prepare input"): # -------------------------------------- Omni-new ------------------------------------------------- self._update_states(scheduler_output) @@ -731,7 +105,7 @@ class NPUARModelRunner(OmniNPUModelRunner): with self.maybe_get_ec_connector_output( scheduler_output, encoder_cache=self.encoder_cache, - ): + ) as ec_connector_output: self._execute_mm_encoder(scheduler_output) return make_empty_encoder_model_runner_output( scheduler_output) @@ -749,21 +123,23 @@ class NPUARModelRunner(OmniNPUModelRunner): if self.dynamic_eplb: self.eplb_updator.forward_before() - (attn_metadata, positions, num_scheduled_tokens_np, - num_input_tokens, num_tokens_across_dp, maybe_padded_num_tokens, - logits_indices, spec_decode_metadata, input_ids, inputs_embeds, - intermediate_tensors, - max_query_len) = (self._prepare_inputs(scheduler_output, - intermediate_tensors)) + (attn_metadata, num_scheduled_tokens_np, num_input_tokens, + num_tokens_across_dp, logits_indices, spec_decode_metadata, + max_query_len) = self._prepare_inputs(scheduler_output) + + (input_ids, inputs_embeds, positions, intermediate_tensors, + model_kwargs, ec_connector_output) = self._preprocess(scheduler_output, + num_input_tokens, + intermediate_tensors) + + # update global cos, sin + update_cos_sin(positions) if self.dynamic_eplb: self.eplb_updator.take_update_info_from_eplb_process() - moe_comm_type = self._select_moe_comm_method(num_input_tokens) # prevent debugger is None - need_dump = self.dump_enable and self.debugger is not None - if need_dump: - assert self.debugger is not None + if self.debugger is not None: dbg_cfg = getattr(self.debugger, "config", None) dump_level = str( getattr(dbg_cfg, "level", @@ -779,7 +155,16 @@ class NPUARModelRunner(OmniNPUModelRunner): has_lora = len(self.input_batch.lora_id_to_lora_request) > 0 aclgraph_runtime_mode, batch_descriptor = \ self.cudagraph_dispatcher.dispatch( - num_tokens=num_input_tokens, uniform_decode=uniform_decode, has_lora=has_lora) + num_tokens=num_input_tokens, + uniform_decode=uniform_decode, + has_lora=has_lora + ) + + if self.ascend_config.enable_async_exponential: + self.sampler.do_async_exponential( + b_s=logits_indices.shape[0], + head_dim=self.model_config.get_vocab_size(), + generators=self.input_batch.sampling_metadata.generators) # Run forward pass with ProfileExecuteDuration().capture_async("forward"): @@ -788,27 +173,23 @@ class NPUARModelRunner(OmniNPUModelRunner): self.vllm_config, num_tokens=num_input_tokens, num_tokens_across_dp=num_tokens_across_dp, - with_prefill=self.with_prefill, - moe_comm_type=moe_comm_type, aclgraph_runtime_mode=aclgraph_runtime_mode, batch_descriptor=batch_descriptor, num_actual_tokens=scheduler_output. total_num_scheduled_tokens, - prefetch_stream=self.prefetch_stream, - model_instance=self.model, - weight_prefetch_method=self.weight_prefetch_method): + model_instance=self.model): self.maybe_setup_kv_connector(scheduler_output) hidden_states = self._generate_process_reqs_hidden_states( - maybe_padded_num_tokens, input_ids, positions, - intermediate_tensors, inputs_embeds) + num_input_tokens, input_ids, positions, + intermediate_tensors, inputs_embeds, model_kwargs) self.maybe_wait_for_kv_save() finished_sending, finished_recving = self.get_finished_kv_transfer( scheduler_output) aux_hidden_states = None - if self.drafter and self.drafter.name == SpecDcodeType.EAGLE3: + if self.use_aux_hidden_state_outputs: hidden_states, aux_hidden_states = hidden_states kv_connector_output = KVConnectorOutput( @@ -818,8 +199,7 @@ class NPUARModelRunner(OmniNPUModelRunner): finished_recving = None with ProfileExecuteDuration().capture_async("post process"): # -------------------------------------- Omni-new ------------------------------------------------- - multimodal_outputs = hidden_states.multimodal_outputs - hidden_states = hidden_states.text_hidden_states + hidden_states, multimodal_outputs = self.extract_multimodal_outputs(hidden_states) if multimodal_outputs is not None: keys_or_type = ( @@ -842,8 +222,8 @@ class NPUARModelRunner(OmniNPUModelRunner): # For mid-pipeline stages, return the hidden states. if not broadcast_pp_output: hidden_states.kv_connector_output = kv_connector_output - if need_dump: - assert self.debugger is not None + self.kv_connector_output = kv_connector_output + if self.debugger is not None: self.debugger.stop() self.debugger.step() return hidden_states @@ -856,20 +236,21 @@ class NPUARModelRunner(OmniNPUModelRunner): pool_output = self._pool( hidden_states, scheduler_output.total_num_scheduled_tokens, - num_scheduled_tokens_np) - if need_dump: - assert self.debugger is not None + num_scheduled_tokens_np, kv_connector_output) + if self.debugger is not None: self.debugger.stop() self.debugger.step() return pool_output - # Sometimes, after the model is compiled through the AOT backend, - # the model output may become a list containing only one Tensor object. - if isinstance(hidden_states, list) and \ - len(hidden_states) == 1 and \ - isinstance(hidden_states[0], torch.Tensor): - hidden_states = hidden_states[0] sample_hidden_states = hidden_states[logits_indices] - logits = self.model.compute_logits(sample_hidden_states) + # -------------------------------------- Omni-new ------------------------------------------------- + # Try with sampling_metadata first; fall back to without for models that don't support it + try: + logits = self.model.compute_logits( + sample_hidden_states, sampling_metadata=self.input_batch.sampling_metadata + ) + except TypeError: + logits = self.model.compute_logits(sample_hidden_states) + # -------------------------------------- Omni-new ------------------------------------------------- if broadcast_pp_output: model_output_broadcast_data = { "logits": logits.contiguous(), @@ -891,18 +272,38 @@ class NPUARModelRunner(OmniNPUModelRunner): kv_connector_output, attn_metadata, positions, - multimodal_outputs # Omni-new + ec_connector_output, + multimodal_outputs, # Omni-new ) + self.kv_connector_output = kv_connector_output return None - @torch.inference_mode + @torch.inference_mode() def sample_tokens( - self, grammar_output: GrammarOutput | None - ) -> ModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors: + self, + grammar_output: GrammarOutput | None = None, + ) -> OmniModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors: + kv_connector_output = self.kv_connector_output + self.kv_connector_output = None + + # -------------------------------------- Omni-new ------------------------------------------------- + kv_extracted_req_ids = getattr(self, "kv_extracted_req_ids", None) + self.kv_extracted_req_ids = None + # -------------------------------------- Omni-new ------------------------------------------------- + if self.execute_model_state is None: # Nothing to do (PP non-final rank case), output isn't used. - return None # noqa - need_dump = self.dump_enable and self.debugger is not None + if not kv_connector_output: + return None # noqa + # In case of PP with kv transfer, we need to pass through the + # kv_connector_output + if kv_connector_output.is_empty(): + return EMPTY_MODEL_RUNNER_OUTPUT + + output = copy(EMPTY_MODEL_RUNNER_OUTPUT) + output.kv_connector_output = kv_connector_output + return output + # Unpack ephemeral state. ( scheduler_output, @@ -910,160 +311,34 @@ class NPUARModelRunner(OmniNPUModelRunner): spec_decode_metadata, hidden_states, sample_hidden_states, - aux_hidden_states, # noqa + aux_hidden_states, kv_connector_output, attn_metadata, positions, - multimodal_outputs, # Omni-new + ec_connector_output, + multimodal_outputs, ) = self.execute_model_state # Clear ephemeral state. self.execute_model_state = None # Apply structured output bitmasks if present. if grammar_output is not None: - logits = self.apply_grammar_bitmask(scheduler_output, - grammar_output, logits) + # here we are different from gpu_model_runner, + # the apply_grammar_bitmask uses torch.compile to optimize this,ascend does not support it now + logits_dtype = logits.dtype + logits = logits.to("cpu").float() + apply_grammar_bitmask(scheduler_output, grammar_output, + self.input_batch, logits) + logits = logits.to(self.device).to(logits_dtype) with ProfileExecuteDuration().capture_async("Sample"): - # Sample the next token and get logprobs if needed. - sampling_metadata = self.input_batch.sampling_metadata - if spec_decode_metadata is None: - if lmhead_tp_enable() and logits is not None: - logits = logits[:self.input_batch.num_reqs] - sampler_output = self.sampler( - logits=logits, - sampling_metadata=sampling_metadata, - ) - else: - if lmhead_tp_enable() and logits is not None: - logits = logits[:len(spec_decode_metadata.logits_indices)] - # When indexing with a tensor (bonus_logits_indices), PyTorch - # creates a new tensor with separate storage from the original - # logits tensor. This means any in-place operations on bonus_logits - # won't affect the original logits tensor. - assert logits is not None - bonus_logits = logits[ - spec_decode_metadata.bonus_logits_indices] - sampler_output = self.sampler( - logits=bonus_logits, - sampling_metadata=sampling_metadata, - ) - bonus_token_ids = sampler_output.sampled_token_ids - - # Just like `bonus_logits`, `target_logits` is a new tensor with - # separate storage from the original `logits` tensor. Therefore, - # it is safe to update `target_logits` in place. - target_logits = logits[ - spec_decode_metadata.target_logits_indices] - output_token_ids = self.rejection_sampler( - spec_decode_metadata, - None, # draft_probs - target_logits, - bonus_token_ids, - sampling_metadata, - ) - sampler_output.sampled_token_ids = output_token_ids - if self.need_accepted_tokens: - self._update_states_after_model_execute(output_token_ids) - discard_sampled_tokens_req_indices = \ - self.discard_request_indices.np[:self.num_discarded_requests] - for i in discard_sampled_tokens_req_indices: - generator = self.input_batch.generators.get(int(i)) - if generator is not None: - generator.set_offset(generator.get_offset() - 4) - - # Copy some objects so they don't get modified after returning. - # This is important when using async scheduling. - req_ids_output_copy = self.input_batch.req_ids.copy() - req_id_to_index_output_copy = \ - self.input_batch.req_id_to_index.copy() - - # NOTE: NPU -> CPU Sync happens here. - # Move as many CPU operations as possible before this sync point. - logprobs_tensors = sampler_output.logprobs_tensors - logprobs_lists = logprobs_tensors.tolists() \ - if logprobs_tensors is not None else None - - # Compute prompt logprobs if needed. - prompt_logprobs_dict = self._get_prompt_logprobs_dict( - hidden_states[:scheduler_output.total_num_scheduled_tokens], - scheduler_output, - ) - - num_sampled_tokens = sampler_output.sampled_token_ids.shape[0] - sampled_token_ids = sampler_output.sampled_token_ids - - if not self.use_async_scheduling: - # Get the valid generated tokens. - max_gen_len = sampled_token_ids.shape[-1] - if max_gen_len == 1: - # No spec decode tokens. It's a tensor. - valid_sampled_token_ids = sampled_token_ids.tolist() - else: - # Includes spec decode tokens. It's a numpy array - valid_sampled_token_ids, _ = self.rejection_sampler.parse_output( - sampled_token_ids, - self.input_batch.vocab_size, - ) - # Mask out the sampled tokens that should not be sampled. - for i in discard_sampled_tokens_req_indices: - valid_sampled_token_ids[int(i)].clear() - else: - valid_sampled_token_ids = [] - invalid_req_indices = discard_sampled_tokens_req_indices.tolist( - ) - invalid_req_indices_set = set(invalid_req_indices) - if self.num_spec_tokens <= 0: - assert sampled_token_ids.shape[-1] == 1 - # Cache the sampled tokens on the NPU and avoid CPU sync. - # These will be copied into input_ids in the next step - # when preparing inputs. - self.input_batch.prev_sampled_token_ids = sampled_token_ids - - - self.input_batch.prev_sampled_token_ids_invalid_indices = \ - invalid_req_indices_set - self.input_batch.prev_req_id_to_index = { - req_id: i - for i, req_id in enumerate(self.input_batch.req_ids) - if i not in invalid_req_indices_set - } - # Cache the sampled tokens in the model runner, so that the scheduler - # doesn't need to send them back. - # NOTE(woosuk): As an exception, when using PP, the scheduler sends - # the sampled tokens back, because there's no direct communication - # between the first-stage worker and the last-stage worker. - for req_idx in range(num_sampled_tokens): - if self.use_async_scheduling: - sampled_ids = [-1] * 1 if \ - req_idx not in invalid_req_indices_set else None - else: - sampled_ids = valid_sampled_token_ids[req_idx] - if not sampled_ids: - continue - - start_idx = self.input_batch.num_tokens_no_spec[req_idx] - end_idx = start_idx + len(sampled_ids) - assert end_idx <= self.model_config.max_model_len, ( - "Sampled token IDs exceed the max model length. " - f"Total number of tokens: {end_idx} > max_model_len: " - f"{self.model_config.max_model_len}") - - self.input_batch.token_ids_cpu[req_idx, - start_idx:end_idx] = sampled_ids - self.input_batch.is_token_ids[req_idx, - start_idx:end_idx] = True - self.input_batch.num_tokens_no_spec[req_idx] = end_idx - self.input_batch.num_tokens[req_idx] = end_idx - req_id = self.input_batch.req_ids[req_idx] - req_state = self.requests[req_id] - req_state.output_token_ids.extend(sampled_ids) + sampler_output = self._sample(logits, spec_decode_metadata) def propose_draft_token_ids(sampled_token_ids): assert self.spec_decode_common_attn_metadata is not None self._draft_token_ids = self.propose_draft_token_ids( sampled_token_ids, - sampling_metadata, + self.input_batch.sampling_metadata, scheduler_output, spec_decode_metadata, positions, @@ -1071,12 +346,30 @@ class NPUARModelRunner(OmniNPUModelRunner): hidden_states, attn_metadata, aux_hidden_states, + sample_hidden_states ) + self._copy_draft_token_ids_to_cpu(scheduler_output) + + ( + logprobs_lists, + valid_sampled_token_ids, + prompt_logprobs_dict, + req_ids_output_copy, + req_id_to_index_output_copy, + invalid_req_indices, + ) = self._bookkeeping_sync( + scheduler_output, + sampler_output, + logits, + hidden_states, + scheduler_output.total_num_scheduled_tokens, + spec_decode_metadata, + ) with ProfileExecuteDuration().capture_async("Draft"): if self.speculative_config: use_padded_batch_for_eagle = self.speculative_config and \ - self.speculative_config.method == "mtp" and \ + self.speculative_config.use_eagle() and \ not self.speculative_config.disable_padded_drafter_batch if use_padded_batch_for_eagle: # EAGLE speculative decoding can use the GPU sampled tokens @@ -1100,7 +393,9 @@ class NPUARModelRunner(OmniNPUModelRunner): dtype=np.int32, ) - self._process_additional_information_updates(hidden_states, multimodal_outputs, num_scheduled_tokens_np) + self._process_additional_information_updates( + hidden_states, multimodal_outputs, num_scheduled_tokens_np, scheduler_output + ) pooler_output: list[dict[str, object]] = [] for rid in req_ids_output_copy: @@ -1143,6 +438,7 @@ class NPUARModelRunner(OmniNPUModelRunner): pooler_output=(pooler_output if self.vllm_config.model_config.engine_output_type != "text" else None), kv_connector_output=kv_connector_output, ) + output.kv_extracted_req_ids = kv_extracted_req_ids # -------------------------------------- Omni-new ------------------------------------------------- durations = ProfileExecuteDuration().pop_captured_sync() @@ -1157,123 +453,248 @@ class NPUARModelRunner(OmniNPUModelRunner): if self.dynamic_eplb: self.eplb_updator.forward_end() if not self.use_async_scheduling: - if need_dump: + if self.debugger is not None: assert self.debugger is not None self.debugger.stop() self.debugger.step() return model_runner_output - if need_dump: + if self.debugger is not None: assert self.debugger is not None self.debugger.stop() self.debugger.step() return AsyncGPUModelRunnerOutput( model_runner_output=model_runner_output, - sampled_token_ids=sampled_token_ids, + sampled_token_ids=sampler_output.sampled_token_ids, logprobs_tensors=sampler_output.logprobs_tensors, invalid_req_indices=invalid_req_indices, async_output_copy_stream=self.async_output_copy_stream, vocab_size=self.input_batch.vocab_size, ) - def _merge_additional_information_update(self, req_id: str, upd: dict) -> None: - req_state = self.requests.get(req_id) - if req_state is None: - return - existing = getattr(req_state, "additional_information_cpu", {}) - if not isinstance(existing, dict): - existing = {} - merged = dict(existing) - for k, v in upd.items(): - if isinstance(v, torch.Tensor): - merged[k] = v.detach().to("cpu").contiguous() - elif isinstance(v, list): - merged[k] = [ - (item.detach().to("cpu").contiguous() if isinstance(item, torch.Tensor) else item) for item in v - ] - else: - merged[k] = v - setattr(req_state, "additional_information_cpu", merged) - - def _generate_process_reqs_hidden_states(self, maybe_padded_num_tokens, + def _generate_process_reqs_hidden_states(self, num_input_tokens, input_ids, positions, intermediate_tensors, - inputs_embeds): + inputs_embeds, model_kwargs): + assert self.model is not None hidden_states = self._model_forward(input_ids=input_ids, positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, - **self._init_model_kwargs()) + **model_kwargs) forward_context = get_forward_context() if forward_context.cudagraph_runtime_mode == CUDAGraphMode.FULL \ and not self.use_sparse: - # TODO: maybe_padded_num_tokens will be removed, use num_input_tokens instead if self.vllm_config.model_config.use_mla: if self.pcp_size * self.dcp_size > 1: - # FIXME: Try using `auto_dispatch_capture=True` - update_mla_attn_dcp_pcp_params(self.update_stream, - forward_context, - maybe_padded_num_tokens) - else: - # FIXME: Try using `auto_dispatch_capture=True` - update_mla_attn_params(self.update_stream, forward_context, - maybe_padded_num_tokens, - self.speculative_config) - else: - if self.pcp_size * self.dcp_size > 1: - update_attn_dcp_pcp_params(self.update_stream, - forward_context, - maybe_padded_num_tokens) - else: - update_attn_params(self.update_stream, forward_context, - maybe_padded_num_tokens) + update_full_graph_params(self.attn_backend, self.update_stream, forward_context, + num_input_tokens, self.vllm_config, + self.vllm_config.speculative_config) if get_forward_context().sp_enabled and not isinstance( hidden_states, IntermediateTensors): - hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) - pad_size = get_forward_context().pad_size - if pad_size > 0: - hidden_states = hidden_states[:-pad_size, :] + hidden_states = self._all_gather_hidden_states_and_aux( + hidden_states) + return hidden_states if self.pcp_size == 1 else self.pcp_manager.get_restore_hidden_states( + hidden_states) - if self.pcp_size > 1: - hidden_states = get_pcp_group().all_gather( - hidden_states[:self.num_actual_tokens_pcp_padded // - self.pcp_size], 0) - hidden_states = torch.index_select( - hidden_states, 0, - self.pcp_allgather_restore_idx[:hidden_states.shape[0]]) - return hidden_states + def _handle_finished_requests_kv_transfer(self, scheduler_output: SchedulerOutput) -> list[str]: + """Handle KV cache transfer for finished requests. - def _process_additional_information_updates( + Returns list of request IDs that were processed (for Scheduler to free blocks). + """ + finished_reqs = getattr(scheduler_output, "finished_requests_needing_kv_transfer", {}) + if not finished_reqs: + return [] + + logger.debug(f"Processing KV transfer for {len(finished_reqs)} requests") + + extracted_ids = [] + for req_id, data in finished_reqs.items(): + try: + seq_len = data.get("seq_len", 0) + block_ids = data.get("block_ids", []) + if not block_ids: + logger.warning(f"Request {req_id} has no block IDs, skipping") + continue + + # Extract KV cache from GPU blocks -> CPU tensors + kv_data = self._extract_kv_cache(req_id, block_ids, seq_len) + if kv_data: + # Transfer to downstream stage via connector + self._transfer_kv_cache(kv_data) + + except Exception as e: + logger.error(f"Failed KV transfer for {req_id}: {e}") + finally: + extracted_ids.append(req_id) + + return extracted_ids + + def _extract_kv_cache(self, req_id: str, block_ids: list[int], seq_len: int) -> KVCacheTransferData | None: + """Extract KV cache from GPU blocks for a single request.""" + num_layers = len(self.kv_caches) + key_cache = [None] * num_layers + value_cache = [None] * num_layers + + for layer_idx, kv_tensor in enumerate(self.kv_caches): + # Validate block IDs + max_block = kv_tensor.shape[1] - 1 + valid_ids = [bid for bid in block_ids if 0 <= bid <= max_block] + if not valid_ids: + continue + + # Extract and reshape: [2, n_blocks, block_size, n_heads, head_dim] + # -> [2, seq_len, n_heads, head_dim] + selected = kv_tensor[:, valid_ids] # [2, n_valid, block_size, n_heads, head_dim] + n_kv, n_blks, blk_sz, n_heads, d_head = selected.shape + flat = selected.reshape(n_kv, n_blks * blk_sz, n_heads, d_head) + if seq_len < flat.shape[1]: + flat = flat[:, :seq_len] + + # Move to CPU + flat_cpu = flat.detach().cpu().contiguous() + key_cache[layer_idx] = flat_cpu[0] + value_cache[layer_idx] = flat_cpu[1] + + if not any(k is not None for k in key_cache): + return None + + return KVCacheTransferData( + request_id=req_id, + layer_blocks={"key_cache": key_cache, "value_cache": value_cache}, + block_ids=block_ids, + metadata={ + "block_size": self.cache_config.block_size, + "num_layers": num_layers, + "dtype": str(self.cache_config.cache_dtype), + "seq_len": seq_len, + }, + ) + + def _transfer_kv_cache(self, kv_data: KVCacheTransferData) -> None: + """Transfer KV cache data to downstream stage via OmniConnector.""" + connector = self._get_or_create_connector() + if not connector: + return + + # Resolve global request ID if available + transfer_req_id = self._resolve_global_request_id(kv_data.request_id) + from_stage, to_stage = self._detect_transfer_stages() + + # Prepare data and transfer with retry + data_dict = kv_data.to_dict() + data_dict["request_id"] = transfer_req_id + + success, size, _ = self._transfer_with_retry( + connector, from_stage, to_stage, f"kv_cache_{transfer_req_id}", data_dict + ) + + if success: + logger.info(f"KV transfer OK: {transfer_req_id}, {size} bytes") + else: + logger.error(f"KV transfer FAILED: {transfer_req_id}") + + def _get_or_create_connector(self) -> Any | None: + """Get existing connector or create one from config.""" + if self.omni_connector: + return self.omni_connector + + from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory + from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec + + config = self._get_omni_connector_config() + if not config or not isinstance(config, dict): + logger.warning("No valid OmniConnector config found") + return None + + c_type = config.get("type") + if not c_type: + logger.error("OmniConnector config missing 'type' field") + return None + + c_extra = {k: v for k, v in config.items() if k != "type"} + self.omni_connector = OmniConnectorFactory.create_connector(ConnectorSpec(name=c_type, extra=c_extra)) + return self.omni_connector + + def _get_omni_connector_config(self) -> dict[str, Any] | None: + """Get OmniConnector configuration from model config.""" + # Primary: omni_kv_config from YAML + omni_kv = getattr(self.model_config, "omni_kv_config", None) + if isinstance(omni_kv, dict): + cfg = omni_kv.get("connector_config") + if isinstance(cfg, dict) and cfg: + return cfg + + # Fallback: kv_transfer_config + kv_cfg = getattr(self.vllm_config, "kv_transfer_config", None) + if kv_cfg: + direct = getattr(kv_cfg, "omni_connector_config", None) + if isinstance(direct, dict) and direct: + return direct + extra = getattr(kv_cfg, "kv_connector_extra_config", None) + if isinstance(extra, dict): + omni = extra.get("omni_connector_config") + if isinstance(omni, dict) and omni: + return omni + + return None + + def _detect_transfer_stages(self) -> tuple[str, str]: + """Detect source and target stages for KV transfer.""" + omni_kv = getattr(self.model_config, "omni_kv_config", None) + if isinstance(omni_kv, dict): + from_s = omni_kv.get("omni_from_stage") + to_s = omni_kv.get("omni_to_stage") + if from_s and to_s: + return str(from_s), str(to_s) + + raise ValueError( + "KV transfer stages not configured. Please set 'omni_from_stage' and 'omni_to_stage' in omni_kv_config." + ) + + def _resolve_global_request_id(self, req_id: str) -> str: + """Resolve global request ID from request state.""" + req_state = self.requests.get(req_id) + if not req_state: + return req_id + + add_info = getattr(req_state, "additional_information_cpu", {}) or {} + global_id = add_info.get("global_request_id") + if global_id: + if isinstance(global_id, list) and global_id: + global_id = global_id[0] + if isinstance(global_id, bytes): + return global_id.decode("utf-8") + return str(global_id) + return req_id + + def _transfer_with_retry( self, - hidden_states: torch.Tensor, - multimodal_outputs: object, - num_scheduled_tokens_np: np.ndarray, - ) -> None: - """Process model-provided per-request additional_information updates and merge into request state.""" - try: - # execute the custom postprocess function - # TODO(Peiqi): do we have a more elegant way to do this? - if hasattr(self.model, "has_postprocess") and self.model.has_postprocess: - for req_index, req_id in enumerate(self.input_batch.req_ids): - req_state = self.requests.get(req_id) - req_infos = ( - getattr(req_state, "additional_information_cpu", None) if req_state is not None else None - ) - start_offset = int(self.query_start_loc.cpu[req_index]) - sched_tokens = int(num_scheduled_tokens_np[req_index]) - s, e = start_offset, start_offset + sched_tokens - # only consider to store data into update dict. - hidden_states_slice = hidden_states[s:e] - update_dict = self.model.postprocess(hidden_states_slice, **req_infos) - self._merge_additional_information_update(req_id, update_dict) - except Exception as e: - logger.error( - f"Error merging for requests:{self.input_batch.req_ids} " - f"additional information update: {e}, with the multimodal_outputs " - f"as {multimodal_outputs}" - ) - import traceback + connector: Any, + from_stage: str, + to_stage: str, + request_id: str, + data: dict[str, Any], + max_retries: int = 3, + ) -> tuple[bool, int, dict[str, Any] | None]: + """Transfer data with retry and exponential backoff.""" + import time - traceback.print_exc() + for attempt in range(max_retries): + try: + put_key = f"omni_{from_stage}_to_{to_stage}_{request_id}" + success, size, metadata = connector.put( + from_stage=from_stage, to_stage=to_stage, put_key=put_key, data=data + ) + if success: + return success, size, metadata + logger.warning(f"Transfer attempt {attempt + 1} failed for {request_id}") + except Exception as e: + logger.warning(f"Transfer attempt {attempt + 1} exception: {e}") + + if attempt < max_retries - 1: + time.sleep(0.1 * (2**attempt)) + + return False, 0, None diff --git a/vllm_omni/worker/npu/npu_ar_worker.py b/vllm_omni/worker/npu/npu_ar_worker.py index fb2f0586e..c4895ee1b 100644 --- a/vllm_omni/worker/npu/npu_ar_worker.py +++ b/vllm_omni/worker/npu/npu_ar_worker.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm_ascend.worker.worker_v1 import NPUWorker +from vllm.v1.worker.workspace import init_workspace_manager +from vllm_ascend.worker.worker import NPUWorker from vllm_omni.worker.npu.npu_ar_model_runner import NPUARModelRunner @@ -10,6 +11,8 @@ class NPUARWorker(NPUWorker): """NPU AR worker for thinker/talker stages in Omni model.""" def init_device(self): - device = self._init_device() + self.device = self._init_device() + num_ubatches = 1 + init_workspace_manager(self.device, num_ubatches) - self.model_runner: NPUARModelRunner = NPUARModelRunner(self.vllm_config, device) + self.model_runner = NPUARModelRunner(self.vllm_config, self.device) diff --git a/vllm_omni/worker/npu/npu_generation_model_runner.py b/vllm_omni/worker/npu/npu_generation_model_runner.py index e6641ca12..e1beef982 100644 --- a/vllm_omni/worker/npu/npu_generation_model_runner.py +++ b/vllm_omni/worker/npu/npu_generation_model_runner.py @@ -5,584 +5,45 @@ from __future__ import annotations import gc import math -from typing import Any +from copy import copy import numpy as np import torch -import torch.nn as nn from vllm.config import CUDAGraphMode from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer +from vllm.distributed.kv_transfer import has_kv_transfer_group from vllm.distributed.parallel_state import get_pp_group from vllm.logger import logger -from vllm.multimodal.inputs import MultiModalKwargs from vllm.sequence import IntermediateTensors from vllm.utils.math_utils import cdiv -from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec -from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT -from vllm.v1.spec_decode.metadata import SpecDecodeMetadata +from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput +from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, AsyncModelRunnerOutput, make_empty_encoder_model_runner_output from vllm.v1.worker.gpu_model_runner import AsyncGPUModelRunnerOutput from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorOutput from vllm_ascend.ascend_forward_context import MoECommType, get_mc2_tokens_capacity, set_ascend_forward_context -from vllm_ascend.attention.attention_v1 import AscendAttentionState -from vllm_ascend.attention.utils import AscendCommonAttentionMetadata -from vllm_ascend.patch.worker.patch_module import patch_torch_npu_argsort +from vllm_ascend.ops.rotary_embedding import update_cos_sin from vllm_ascend.platform import NPUPlatform -from vllm_ascend.spec_decode.interface import SpecDcodeType from vllm_ascend.utils import ProfileExecuteDuration, enable_sp, lmhead_tp_enable from vllm_omni.outputs import OmniModelRunnerOutput +from vllm_omni.worker.npu.npu_ar_model_runner import ExecuteModelState from vllm_omni.worker.npu.npu_model_runner import OmniNPUModelRunner class NPUGenerationModelRunner(OmniNPUModelRunner): """Generation model runner for vLLM-omni on NPU (non-autoregressive).""" - def _prepare_inputs( - self, - scheduler_output: SchedulerOutput, - intermediate_tensors: IntermediateTensors | None = None, - ) -> tuple[ - dict[str, Any], - torch.Tensor, - np.ndarray, - int, - torch.Tensor, - int, - torch.Tensor, - SpecDecodeMetadata, - torch.Tensor | None, - torch.Tensor | None, - torch.Tensor | None, - int, - ]: - total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens - assert total_num_scheduled_tokens > 0 - num_reqs = self.input_batch.num_reqs - assert num_reqs > 0 - - # OPTIMIZATION: Start copying the block table first. - # This way, we can overlap the copy with the following CPU operations. - self.input_batch.block_table.commit_block_table(num_reqs) - - # Get the number of scheduled tokens for each request. - req_ids = self.input_batch.req_ids - tokens = [scheduler_output.num_scheduled_tokens[i] for i in req_ids] - num_scheduled_tokens = np.array(tokens, dtype=np.int32) - - req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) - _, arange = self._get_cumsum_and_arange(num_scheduled_tokens) - positions_np = np.add( - self.input_batch.num_computed_tokens_cpu[req_indices], - arange, - ) - - self.input_batch.block_table.compute_slot_mapping(req_indices, positions_np) - self.input_batch.block_table.commit_slot_mapping(total_num_scheduled_tokens) - if self.pcp_size > 1: - if not self.vllm_config.model_config.use_mla: - self.generate_kv_idx(scheduler_output) - tokens, position_pcp, pcp_unpad_mask = self._update_tokens_for_pcp(tokens) - num_scheduled_tokens = np.array(tokens, dtype=np.int32) - total_num_scheduled_tokens = sum(num_scheduled_tokens[:num_reqs]) - else: - position_pcp, pcp_unpad_mask = None, None - self.num_pcp_pads = self.num_pcp_pads[:num_reqs] - - total_num_pcp_pads = sum(self.num_pcp_pads) - max_num_scheduled_tokens = max(tokens) - num_valid_tokens = np.array( - [ - num_tokens - len(scheduler_output.scheduled_spec_decode_tokens.get(i, [])) - for num_tokens, i in zip(tokens, req_ids) - ], - dtype=np.int32, - ) - - if self.use_aclgraph and total_num_scheduled_tokens <= self.cudagraph_batch_sizes[-1]: - # Add padding to the batch size. - num_input_tokens = self.vllm_config.pad_for_cudagraph(total_num_scheduled_tokens) - elif self.use_aclgraph and enable_sp(self.vllm_config): - # When using aclgraph, if total_num_scheduled_tokens exceeds the maximum graph size, - # the model will fall back to running its FX graph in eager mode. - # In this case, when sequence parallelism is enabled, we need to pad tokens to align - # with tp_size because pad_size cannot be captured by the FX graph - tp_size = self.vllm_config.parallel_config.tensor_parallel_size - num_input_tokens = math.ceil(total_num_scheduled_tokens / tp_size) * tp_size - else: - # Eager mode. - num_input_tokens = total_num_scheduled_tokens - - # Get the attention state. - attn_state = self._build_attn_state(num_reqs, num_scheduled_tokens, num_valid_tokens) - self.attn_state = attn_state # type: ignore - - # Determine if it's a splitfuse batch - with_prefill = attn_state not in [AscendAttentionState.DecodeOnly, AscendAttentionState.SpecDecoding] - - self.query_lens = torch.from_numpy(num_scheduled_tokens) - - # Get info across DP ranks. - # NOTE: maybe_padded_num_tokens is only used when using TorchAir with DP, - # Otherwise, it's just max_tokens_across_dp_cpu - (maybe_padded_num_tokens, num_tokens_across_dp, with_prefill) = self._sync_metadata_across_dp( - num_input_tokens, with_prefill - ) - - # TODO: Now that num_input_tokens is basically identical with maybe_padded_num_tokens - # We should consider removing maybe_padded_num_tokens later - num_input_tokens = maybe_padded_num_tokens - - # Hot-Swap lora model - if self.lora_config: - self.set_active_loras(self.input_batch, num_scheduled_tokens) - - # Get request indices. - # E.g., [2, 5, 3] -> [0, 0, 1, 1, 1, 1, 1, 2, 2, 2] - req_indices = np.repeat(self.arange_np[:num_reqs], num_scheduled_tokens) - - # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] - # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - cu_num_tokens, arange = self._get_cumsum_and_arange(num_scheduled_tokens) - - if self.pcp_size > 1: - positions_np = self.positions.np[:total_num_scheduled_tokens] - np.add( - self.input_batch.num_computed_tokens_cpu[req_indices], - position_pcp[:total_num_scheduled_tokens], - out=positions_np, - ) - else: - self.positions.np[:total_num_scheduled_tokens] = positions_np - - # Calculate M-RoPE positions. - # Only relevant for models using M-RoPE (e.g, Qwen2-VL) - if self.uses_mrope: - self._calc_mrope_positions(scheduler_output) - - # Only relevant for models using M-RoPE (e.g, Qwen2-VL) - self.mrope_positions.gpu[:, :total_num_scheduled_tokens].copy_( - self.mrope_positions.cpu[:, :total_num_scheduled_tokens], non_blocking=True - ) - - # Get token indices. - # E.g., [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - # -> [0, 1, M, M + 1, M + 2, M + 3, M + 4, 2 * M, 2 * M + 1, 2 * M + 2] - # where M is the max_model_len. - token_indices = positions_np + req_indices * self.input_batch.token_ids_cpu.shape[1] - token_indices_tensor = torch.from_numpy(token_indices) - # Prepare input_ids. - # NOTE(woosuk): We use torch.index_select instead of np.take here - # because torch.index_select is much faster than np.take for large - # tensors. - torch.index_select( - self.input_batch.token_ids_cpu_tensor.flatten(), - 0, - token_indices_tensor, - out=self.input_ids.cpu[:total_num_scheduled_tokens], - ) - if self.enable_prompt_embeds: - is_token_ids = self.input_batch.is_token_ids_tensor.flatten() - torch.index_select( - is_token_ids, 0, token_indices_tensor, out=self.is_token_ids.cpu[:total_num_scheduled_tokens] - ) - - # Because we did not pre-allocate a massive prompt_embeds CPU tensor on - # the InputBatch, we need to fill in the prompt embeds into the expected - # spots in the GpuModelRunner's pre-allocated prompt_embeds tensor. - if self.input_batch.req_prompt_embeds and (self.is_multimodal_model or self.enable_prompt_embeds): - output_idx = 0 - for req_idx in range(num_reqs): - num_sched = num_scheduled_tokens[req_idx] - - # Skip if this request doesn't have embeddings - if req_idx not in self.input_batch.req_prompt_embeds: - output_idx += num_sched - continue - - # Skip if no tokens scheduled - if num_sched <= 0: - output_idx += num_sched - continue - - req_embeds = self.input_batch.req_prompt_embeds[req_idx] - start_pos = self.input_batch.num_computed_tokens_cpu[req_idx] - - # Skip if trying to read beyond available embeddings - if start_pos >= req_embeds.shape[0]: - output_idx += num_sched - continue - - # Copy available embeddings - end_pos = start_pos + num_sched - actual_end = min(end_pos, req_embeds.shape[0]) - actual_num_sched = actual_end - start_pos - - if actual_num_sched > 0: - self.inputs_embeds.cpu[output_idx : output_idx + actual_num_sched].copy_( - req_embeds[start_pos:actual_end] - ) - - output_idx += num_sched - - self.query_start_loc.np[0] = 0 - self.query_start_loc.np[1 : num_reqs + 1] = cu_num_tokens - self.query_start_loc.copy_to_gpu() - - self.seq_lens.np[:num_reqs] = self.input_batch.num_computed_tokens_cpu[:num_reqs] + num_scheduled_tokens - self.seq_lens.copy_to_gpu() - - # Fill unused with -1. Needed for reshape_and_cache - self.query_start_loc.gpu[num_reqs + 1 :].fill_(-1) - self.seq_lens.gpu[num_reqs:].fill_(0) - - self.query_lens = torch.from_numpy(num_scheduled_tokens) - - # Copy the tensors to the NPU. - self._prepare_input_ids(scheduler_output, total_num_scheduled_tokens, cu_num_tokens) - self.positions.cpu[total_num_scheduled_tokens:num_input_tokens].zero_() - self.positions.copy_to_gpu() - - attn_state = self._build_attn_state(num_reqs, num_scheduled_tokens, num_valid_tokens) - self.attn_mask = self._make_attention_mask(attn_state) - self.attn_state = attn_state # type: ignore - - self.with_prefill = with_prefill - self.num_tokens_across_dp = num_tokens_across_dp - attn_metadata: dict[str, Any] = {} - - # Record the index of requests that should not be sampled, - # so that we could clear the sampled tokens before returning - num_tokens = [self.requests[r].num_tokens for r in self.input_batch.req_ids] - num_tokens_np = np.array(num_tokens, dtype=np.int32) - num_reqs = self.input_batch.num_reqs - if self.pcp_size > 1: - # while pcp > 1, we need the original num_scheduled_tokens before split - # to calculate discard_requests_mask - tokens_original = [scheduler_output.num_scheduled_tokens[i] for i in req_ids] - original_seq_lens_np = self.input_batch.num_computed_tokens_cpu[:num_reqs] + np.array( - tokens_original, dtype=np.int32 - ) - discard_requests_mask = original_seq_lens_np < num_tokens_np - else: - discard_requests_mask = self.seq_lens.np[:num_reqs] < num_tokens_np - - discard_request_indices = np.nonzero(discard_requests_mask)[0] - self.num_discarded_requests = len(discard_request_indices) - self.discard_request_indices.np[: self.num_discarded_requests] = discard_request_indices - self.discard_request_indices.copy_to_gpu(self.num_discarded_requests) - - # _prepare_inputs may reorder the batch, so we must gather - # multi-modal outputs after that to ensure the correct order - if self.is_multimodal_model: - with self.maybe_get_ec_connector_output( - scheduler_output, - encoder_cache=self.encoder_cache, - ): - # Run the multimodal encoder if any. - self._execute_mm_encoder(scheduler_output) - - # NOTE(woosuk): To unify token ids and soft tokens (vision - # embeddings), we always use embeddings (rather than token ids) - # as input to the multimodal model, even when the input is text. - input_ids = self.input_ids.gpu[:total_num_scheduled_tokens] - mm_embeds, is_mm_embed = self._gather_mm_embeddings(scheduler_output) - - inputs_embeds = self.model.embed_input_ids( - input_ids, - multimodal_embeddings=mm_embeds, - is_multimodal=is_mm_embed, - ) - - # TODO(woosuk): Avoid the copy. Optimize. - self.inputs_embeds.gpu[:total_num_scheduled_tokens].copy_(inputs_embeds) - inputs_embeds = self.inputs_embeds.gpu[:num_input_tokens] - # -------------------------------------- Omni-new ------------------------------------------------- - # NOTE(gcanlin): We don't set input_ids to None in vllm-omni. - model_kwargs = { - **self._init_model_kwargs(), - **self._extract_mm_kwargs(scheduler_output), - } - # -------------------------------------- Omni-new ------------------------------------------------- - elif self.enable_prompt_embeds and get_pp_group().is_first_rank: - # Get the input embeddings for the tokens that are not input embeds, - # then put them into the appropriate positions. - # TODO(qthequartermasterman): Since even when prompt embeds are - # enabled, (a) not all requests will use prompt embeds, and (b) - # after the initial prompt is processed, the rest of the generated - # tokens will be token ids, it is not desirable to have the - # embedding layer outside of the acl graph all the time. The v0 - # engine avoids this by "double compiling" the acl graph, once - # with input_ids and again with inputs_embeds, for all num_tokens. - # If a batch only has token ids, then including the embedding layer - # in the acl graph will be more performant (like in the else case - # below). - token_ids_idx = self.is_token_ids.gpu[:total_num_scheduled_tokens].nonzero(as_tuple=False).squeeze(1) - # Some tokens ids may need to become embeds - if token_ids_idx.numel() > 0: - token_ids = self.input_ids.gpu[token_ids_idx] - tokens_to_embeds = self.model.embed_input_ids(input_ids=token_ids) - self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds - - inputs_embeds = self.inputs_embeds.gpu[:num_input_tokens] - input_ids = None - else: - # For text-only models, we use token ids as input. - # While it is possible to use embeddings as input just like the - # multimodal models, it is not desirable for performance since - # then the embedding layer is not included in the ACL graph. - input_ids = self.input_ids.gpu[:num_input_tokens] - inputs_embeds = None - positions = self.positions.gpu[:num_input_tokens] - if self.uses_mrope: - positions = self.mrope_positions.gpu[:, :num_input_tokens] - - # type: ignore - if get_pp_group().is_first_rank: - intermediate_tensors = None - else: - assert intermediate_tensors is not None - assert self.intermediate_tensors is not None - # If both flashcomm1 and pp are used simultaneously, - # the shape of the received data and the shape of the space to be copied to will not match, - # requiring a recalculation of the incoming data's shape. - tp_size = get_tensor_model_parallel_world_size() - num_input_tokens_with_flashcomm1 = num_input_tokens - if enable_sp(): - num_input_tokens_with_flashcomm1 = (num_input_tokens + tp_size - 1) // tp_size - for k, v in intermediate_tensors.items(): - self.intermediate_tensors[k][:num_input_tokens_with_flashcomm1].copy_( - v[:num_input_tokens_with_flashcomm1], non_blocking=True - ) - intermediate_tensors = IntermediateTensors( - {k: v[:num_input_tokens_with_flashcomm1] for k, v in self.intermediate_tensors.items()} - ) - - use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0 - if not use_spec_decode: - # NOTE(woosuk): Due to chunked prefills, the batch may contain - # partial requests. While we should not sample any token - # from these partial requests, we do so for simplicity. - # We will ignore the sampled tokens from the partial requests. - # TODO: Support prompt logprobs. - spec_decode_metadata = None - if self.pcp_size * self.dcp_size > 1: - logits_indices = torch.from_numpy(cu_num_tokens) * self.pcp_size - self.num_pcp_pads[:num_reqs] - 1 - logits_indices = logits_indices.pin_memory().to(self.device, non_blocking=True) - else: - logits_indices = self.query_start_loc.gpu[1 : num_reqs + 1] - 1 - else: - # Get the number of draft tokens for each request. - # Iterate over the dictionary rather than all requests since not all - # requests have draft tokens. - num_draft_tokens = np.zeros(num_reqs, dtype=np.int32) - # For chunked prefills, use -1 as mask rather than 0, as guided - # decoding may rollback speculative tokens. - num_decode_draft_tokens = np.full(num_reqs, -1, dtype=np.int32) - for req_id, draft_token_ids in scheduler_output.scheduled_spec_decode_tokens.items(): - req_idx = self.input_batch.req_id_to_index[req_id] - num_draft_tokens[req_idx] = len(draft_token_ids) - num_decode_draft_tokens[req_idx] = ( - len(draft_token_ids) - if ( - self.input_batch.num_computed_tokens_cpu[req_idx] >= self.input_batch.num_prompt_tokens[req_idx] - ) - else -1 - ) - - spec_decode_metadata = self._calc_spec_decode_metadata( - num_draft_tokens, cu_num_tokens, self.num_pcp_pads[:num_reqs] - ) - logits_indices = spec_decode_metadata.logits_indices - - # For DECODE only cuda graph of some attention backends (e.g., GDN). - self.num_decode_draft_tokens.np[:num_reqs] = num_decode_draft_tokens - self.num_decode_draft_tokens.np[num_reqs:].fill(-1) - self.num_decode_draft_tokens.copy_to_gpu() - # save logits_indices for pcp spec decode usage - self.logits_indices = logits_indices - - # Used in the below loop. - # query_start_loc_cpu = self.query_start_loc.cpu[:num_reqs + 1] - num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[:num_reqs] - self.spec_decode_common_attn_metadata = None - if use_spec_decode and self.need_accepted_tokens: - self.num_accepted_tokens.np[:num_reqs] = self.input_batch.num_accepted_tokens_cpu[:num_reqs] - self.num_accepted_tokens.np[num_reqs:].fill(1) - self.num_accepted_tokens.copy_to_gpu() - - if self.speculative_config and self.pcp_size > 1: - self._generate_pcp_mtp_input( - num_reqs, scheduler_output.total_num_scheduled_tokens, scheduler_output.num_scheduled_tokens - ) - - long_seq_metadata = self._generate_pcp_metadata(total_num_scheduled_tokens) - # Prepare the attention metadata for each KV cache group and make layers - # in the same group share the same metadata. - for kv_cache_group_id, kv_cache_group_spec in enumerate(self.kv_cache_config.kv_cache_groups): - # NOTE: This is strange, why did we use total_num_scheduled_tokens before? - slot_mapping_size = ( - total_num_scheduled_tokens - if self.pcp_size == 1 - else total_num_scheduled_tokens * self.pcp_size - total_num_pcp_pads - ) - if isinstance(kv_cache_group_spec.kv_cache_spec, EncoderOnlyAttentionSpec): - # Encoder-only layers do not have KV cache, so we need to - # create a dummy block table and slot mapping for them. - blk_table_tensor = torch.zeros( - (num_reqs, 1), - dtype=torch.int32, - device=self.device, - ) - slot_mapping = torch.zeros( - (total_num_scheduled_tokens,), - dtype=torch.int64, - device=self.device, - ) - else: - blk_table = self.input_batch.block_table[kv_cache_group_id] - blk_table_tensor = blk_table.get_device_tensor() - blk_table.slot_mapping.gpu[slot_mapping_size:].fill_(0) - if self.pcp_size > 1: - slot_mapping_for_pcp = blk_table.slot_mapping.gpu[: long_seq_metadata.num_actual_tokens_pcp_padded] - slot_mapping_for_pcp[slot_mapping_size:].fill_(-1) - assert pcp_unpad_mask is not None - pcp_padded_slot_mapping = self.pcp_padded_slot_mapping[: pcp_unpad_mask.shape[0]] - pcp_padded_slot_mapping.fill_(-1) - pcp_padded_slot_mapping[pcp_unpad_mask] = slot_mapping_for_pcp[:slot_mapping_size] - slot_mapping_for_pcp[: long_seq_metadata.num_actual_tokens_pcp_padded] = pcp_padded_slot_mapping - blk_table.slot_mapping.gpu[: long_seq_metadata.num_actual_tokens_pcp_padded] = slot_mapping_for_pcp - slot_mapping = blk_table.slot_mapping.gpu - - # NOTE: This is a temporary hack, now in GPUModelRunner, this prepare_inputs - # has been split to multiple parts, and there are 3 parts that is related to this - # `num_reqs`, we'll take `query_start_loc` as an example: - # 1. self.query_start_loc.np[1 : num_reqs + 1] = cu_num_tokens - # 2. get `num_reqs_padded`, this depends on dispatcher and which is why we have the - # following simplified `dispatch` logic here, we try to minimize the impact - # 3. query_start_loc = self.query_start_loc.gpu[: num_reqs_padded + 1] - uniform_decode = (max_num_scheduled_tokens == self.uniform_decode_query_len) and ( - total_num_scheduled_tokens == max_num_scheduled_tokens * num_reqs - ) - - # TODO: We should make this official ASAP. Also note that if we pad here, - # the builders won’t need to add any extra padding. - if self.compilation_config.cudagraph_mode.decode_mode() == CUDAGraphMode.FULL and uniform_decode: - num_reqs_padded = num_input_tokens // self.uniform_decode_query_len - pad_size = num_reqs_padded - num_reqs - if pad_size > 0: - last_query_loc = self.query_start_loc.gpu[num_reqs] - - steps = torch.arange(1, pad_size + 1, device=self.device, dtype=self.query_start_loc.gpu.dtype) - fill_values = last_query_loc + (steps * self.uniform_decode_query_len) - - self.query_start_loc.gpu[num_reqs + 1 : num_reqs_padded + 1] = fill_values - # So we are trying to simulate the behavior of GPUModelRunner's - # prepare_inputs for uniform decode mode by padding query_start_loc - num_reqs = num_reqs_padded - - # Make AscendCommonAttentionMetadata - common_attn_metadata = AscendCommonAttentionMetadata( - query_start_loc=self.query_start_loc.gpu[: num_reqs + 1], - query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs + 1], - seq_lens_cpu=self.seq_lens.cpu[:num_reqs], - seq_lens=self.seq_lens.gpu[:num_reqs], - num_reqs=num_reqs, - num_actual_tokens=slot_mapping_size, - num_input_tokens=num_input_tokens, - actual_seq_lengths_q=self.actual_seq_lengths_q, - # TODO: change this to the right block table for linear attn - block_table_tensor=blk_table_tensor[:num_reqs], - slot_mapping=slot_mapping, - num_computed_tokens_cpu=num_computed_tokens_cpu, - positions=self.positions.gpu, - attn_mask=self.attn_mask, - spec_attn_mask=self.spec_attn_mask, - attn_state=self.attn_state, - is_only_prefill=bool(np.all(num_valid_tokens != 1)), - max_query_len=max_num_scheduled_tokens, - decode_token_per_req=self.decode_token_per_req, - prefill_context_parallel_metadata=long_seq_metadata, - ) - - if self.speculative_config and self.pcp_size > 1: - # For pcp + spec decode, we flatten block_table - # to avoid irregular spec_attn_mask shape, e.g., - # num_decode_req=2, num_prefill_req=3, num_speculative_tokens=1, - # ori block_table: # [d0, d1, p0, p1, p2] - # (num_reqs_d + num_reqs_p, max_num_blocks), - # flattened block_table: [d0, d0, d1, d1, p0, p1, p2] - # (num_reqs_d * decode_threshold + num_reqs_p, max_num_blocks), - ori_query_lens = ( - self.query_start_loc_pcp_full.cpu[1 : num_reqs + 1] - self.query_start_loc_pcp_full.cpu[:num_reqs] - ) - num_prefill_reqs = (ori_query_lens > self.decode_threshold).sum().item() - num_decode_reqs = num_reqs - num_prefill_reqs - num_decode_reqs_flatten = num_decode_reqs * self.decode_threshold - blk_table_tensor[num_decode_reqs_flatten : num_decode_reqs_flatten + num_prefill_reqs].copy_( - blk_table_tensor[num_decode_reqs : num_decode_reqs + num_prefill_reqs].clone() - ) - blk_table_tensor[:num_decode_reqs_flatten].copy_( - blk_table_tensor[:num_decode_reqs].repeat_interleave(self.decode_threshold, dim=0) - ) - common_attn_metadata.block_table_tensor = blk_table_tensor[: num_decode_reqs_flatten + num_prefill_reqs] - - if self.speculative_config and self.spec_decode_common_attn_metadata is None: - self.spec_decode_common_attn_metadata = common_attn_metadata - - for attn_group in self.attn_groups[kv_cache_group_id]: - common_prefix_len = 0 - extra_attn_metadata_args = {} - builder = attn_group.get_metadata_builder() - if isinstance(builder, GDNAttentionMetadataBuilder): - if use_spec_decode: - patch_torch_npu_argsort() - extra_attn_metadata_args = dict( - num_accepted_tokens=self.num_accepted_tokens.gpu[:num_reqs], - num_decode_draft_tokens_cpu=self.num_decode_draft_tokens.cpu[:num_reqs], - ) - attn_metadata_i = builder.build( - common_prefix_len=common_prefix_len, - common_attn_metadata=common_attn_metadata, - **extra_attn_metadata_args, - ) - elif self.model_config.runner_type == "pooling": - attn_metadata_i = builder.build( - common_prefix_len=common_prefix_len, - common_attn_metadata=common_attn_metadata, - **extra_attn_metadata_args, - ) - else: - attn_metadata_i = builder.build( - common_prefix_len=common_prefix_len, - common_attn_metadata=common_attn_metadata, - model=self.get_model(), - **extra_attn_metadata_args, - ) - - for layer_name in attn_group.layer_names: - attn_metadata[layer_name] = attn_metadata_i - - if lmhead_tp_enable(): - max_num_reqs_across_dp = self.max_num_reqs * self.uniform_decode_query_len - logits_indices = nn.functional.pad(logits_indices, (0, max_num_reqs_across_dp - logits_indices.shape[0])) - - return ( - attn_metadata, - positions, - num_scheduled_tokens, - num_input_tokens, - num_tokens_across_dp, - maybe_padded_num_tokens, - logits_indices, - spec_decode_metadata, - input_ids, - inputs_embeds, - intermediate_tensors, - max_num_scheduled_tokens, - model_kwargs, - ) + def _update_request_states(self, scheduler_output: SchedulerOutput): + cached_reqs = scheduler_output.scheduled_cached_reqs + for _, req_id in enumerate(cached_reqs.req_ids): + req_state = self.requests.get(req_id) + assert req_state is not None + req_state.prompt_token_ids = cached_reqs.prompt_token_ids.get(req_id) + self.input_batch.remove_request(req_id) + # update the request state in self.input_batch + self.input_batch.add_request(req_state) + self._init_mrope_positions(req_state) @torch.inference_mode() def execute_model( @@ -590,38 +51,55 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): scheduler_output: SchedulerOutput, intermediate_tensors: IntermediateTensors | None = None, ) -> OmniModelRunnerOutput | IntermediateTensors: + if self.execute_model_state is not None: + raise RuntimeError("State error: sample_tokens() must be called after execute_model() returns None.") + with ProfileExecuteDuration().capture_async("prepare input"): + # -------------------------------------- Omni-new ------------------------------------------------- + if self.model_config.async_chunk: + self._update_request_states(scheduler_output) + # -------------------------------------- Omni-new ------------------------------------------------- self._update_states(scheduler_output) + if has_ec_transfer() and get_ec_transfer().is_producer: + with self.maybe_get_ec_connector_output( + scheduler_output, + encoder_cache=self.encoder_cache, + ) as ec_connector_output: + self._execute_mm_encoder(scheduler_output) + return make_empty_encoder_model_runner_output(scheduler_output) + if not scheduler_output.total_num_scheduled_tokens: - return EMPTY_MODEL_RUNNER_OUTPUT + if not has_kv_transfer_group(): + logger.debug("skip this step for we receive the data from remote disaggregate prefill node") + # Return empty ModelRunnerOutput if there's no work to do. + return EMPTY_MODEL_RUNNER_OUTPUT + return self.kv_connector_no_forward(scheduler_output, self.vllm_config) if self.dynamic_eplb: self.eplb_updator.forward_before() ( attn_metadata, - positions, num_scheduled_tokens_np, num_input_tokens, num_tokens_across_dp, - maybe_padded_num_tokens, logits_indices, spec_decode_metadata, - input_ids, - inputs_embeds, - intermediate_tensors, max_query_len, - model_kwargs, - ) = self._prepare_inputs(scheduler_output, intermediate_tensors) + ) = self._prepare_inputs(scheduler_output) + + (input_ids, inputs_embeds, positions, intermediate_tensors, model_kwargs, ec_connector_output) = ( + self._preprocess(scheduler_output, num_input_tokens, intermediate_tensors) + ) + + # update global cos, sin + update_cos_sin(positions) if self.dynamic_eplb: self.eplb_updator.take_update_info_from_eplb_process() - moe_comm_type = self._select_moe_comm_method(num_input_tokens) # prevent debugger is None - need_dump = self.dump_enable and self.debugger is not None - if need_dump: - assert self.debugger is not None + if self.debugger is not None: dbg_cfg = getattr(self.debugger, "config", None) dump_level = str(getattr(dbg_cfg, "level", "L1")).upper() if dbg_cfg is not None else "L1" if dump_level in ("L0", "MIX"): @@ -637,6 +115,13 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): num_tokens=num_input_tokens, uniform_decode=uniform_decode, has_lora=has_lora ) + if self.ascend_config.enable_async_exponential: + self.sampler.do_async_exponential( + b_s=logits_indices.shape[0], + head_dim=self.model_config.get_vocab_size(), + generators=self.input_batch.sampling_metadata.generators, + ) + # Run forward pass with ProfileExecuteDuration().capture_async("forward"): with set_ascend_forward_context( @@ -644,23 +129,19 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): self.vllm_config, num_tokens=num_input_tokens, num_tokens_across_dp=num_tokens_across_dp, - with_prefill=self.with_prefill, - moe_comm_type=moe_comm_type, aclgraph_runtime_mode=aclgraph_runtime_mode, batch_descriptor=batch_descriptor, num_actual_tokens=scheduler_output.total_num_scheduled_tokens, - prefetch_stream=self.prefetch_stream, model_instance=self.model, - weight_prefetch_method=self.weight_prefetch_method, ): self.maybe_setup_kv_connector(scheduler_output) # -------------------------------------- Omni-new ------------------------------------------------- - outputs = self._run_generation( + outputs = self._run_generation_model( input_ids=input_ids, positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, - multimodal_kwargs=model_kwargs, + model_kwargs=model_kwargs, logits_indices=logits_indices, ) # -------------------------------------- Omni-new ------------------------------------------------- @@ -669,15 +150,70 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): finished_sending, finished_recving = self.get_finished_kv_transfer(scheduler_output) aux_hidden_states = None - if self.drafter and self.drafter.name == SpecDcodeType.EAGLE3: - hidden_states, aux_hidden_states = outputs + if self.use_aux_hidden_state_outputs: + outputs, aux_hidden_states = outputs kv_connector_output = KVConnectorOutput(finished_sending=finished_sending, finished_recving=finished_recving) finished_sending = None finished_recving = None - # -------------------------------------- Omni-new ------------------------------------------------- - # We don't need any post-process for generation model outputs + _, multimodal_outputs = self.extract_multimodal_outputs(outputs) + # Apply structured output bitmasks if present + self.execute_model_state = ExecuteModelState( + scheduler_output, + None, + spec_decode_metadata, + outputs, + None, + aux_hidden_states, + kv_connector_output, + attn_metadata, + positions, + ec_connector_output, + multimodal_outputs, + ) + self.kv_connector_output = kv_connector_output + return None + + @torch.inference_mode() + def sample_tokens( + self, + grammar_output: GrammarOutput | None = None, + ) -> OmniModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors: + kv_connector_output = self.kv_connector_output + self.kv_connector_output = None + + if self.execute_model_state is None: + # Nothing to do (PP non-final rank case), output isn't used. + if not kv_connector_output: + return None # noqa + # In case of PP with kv transfer, we need to pass through the + # kv_connector_output + if kv_connector_output.is_empty(): + return EMPTY_MODEL_RUNNER_OUTPUT + + output = copy(EMPTY_MODEL_RUNNER_OUTPUT) + output.kv_connector_output = kv_connector_output + return output + + # Unpack ephemeral state. + ( + scheduler_output, + logits, + spec_decode_metadata, + hidden_states, + sample_hidden_states, + aux_hidden_states, + kv_connector_output, + attn_metadata, + positions, + ec_connector_output, + multimodal_outputs, + ) = self.execute_model_state + # Clear ephemeral state. + self.execute_model_state = None + + # -------------------------------------- Omni-new ------------------------------------------------- pooler_output: list[object] = [] if isinstance(multimodal_outputs, torch.Tensor): assert multimodal_outputs.shape[0] == 1, ( @@ -695,8 +231,11 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): {"model_outputs": out.detach().to("cpu").contiguous() if out is not None else None} ) elif isinstance(multimodal_outputs, dict): + mm_payload = {} for key, out in multimodal_outputs.items(): - pooler_output.append({key: out.detach().to("cpu").contiguous() if out is not None else None}) + if out is not None and isinstance(out, torch.Tensor): + mm_payload[key] = out.detach().to("cpu").contiguous() + pooler_output.append(mm_payload) else: raise RuntimeError("Unsupported diffusion output type") output = OmniModelRunnerOutput( @@ -708,26 +247,28 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): pooler_output=pooler_output, kv_connector_output=kv_connector_output, num_nans_in_logits={}, + ec_connector_output=ec_connector_output if self.supports_mm_inputs else None, ) # -------------------------------------- Omni-new ------------------------------------------------- if not self.use_async_scheduling: return output - return AsyncGPUModelRunnerOutput( model_runner_output=output, - sampled_token_ids=[], + sampled_token_ids=torch.tensor([], device=self.device), + logprobs_tensors=None, invalid_req_indices=[], async_output_copy_stream=self.async_output_copy_stream, + vocab_size=self.input_batch.vocab_size, ) - def _run_generation( + def _run_generation_model( self, *, - input_ids: torch.Tensor, - positions: torch.Tensor, + input_ids: torch.Tensor | None, + positions: torch.Tensor | None, intermediate_tensors: IntermediateTensors | None, inputs_embeds: torch.Tensor | None, - multimodal_kwargs: dict, + model_kwargs: dict, logits_indices: torch.Tensor, ) -> torch.Tensor | list[torch.Tensor]: """Run generation from codec codes to waveforms. @@ -739,12 +280,13 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): Returns: Audio waveforms: [batch, 1, waveform_len] or list of tensors """ + # Keep inputs identical to AR runner kwargs = dict( input_ids=input_ids, positions=positions, intermediate_tensors=intermediate_tensors, inputs_embeds=inputs_embeds, - **MultiModalKwargs.as_kwargs(multimodal_kwargs, device=self.device), + **model_kwargs, sampling_metadata=self.input_batch.sampling_metadata, logits_index=logits_indices, sampler=self.sampler, @@ -754,7 +296,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): return self._model_forward(**kwargs) raise RuntimeError( - "The loaded model does not expose generation interfaces 'sample', " + "The loaded model does not expose diffusion interfaces 'sample', " "'forward', or 'diffuse'. Please implement one of them or adapt the runner." ) @@ -768,12 +310,18 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): self, num_tokens: int, with_prefill: bool = False, - aclgraph_runtime_mode: CUDAGraphMode | None = None, + cudagraph_runtime_mode: CUDAGraphMode | None = None, force_attention: bool = False, uniform_decode: bool = False, + is_profile: bool = False, + allow_microbatching: bool = True, + skip_eplb: bool = False, + remove_lora: bool = True, + activate_lora: bool = False, + is_graph_capturing: bool = False, ) -> torch.Tensor: # only support eager mode and piecewise graph now - assert aclgraph_runtime_mode is None or aclgraph_runtime_mode in { + assert cudagraph_runtime_mode is None or cudagraph_runtime_mode in { CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL, @@ -788,8 +336,15 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): if self.is_kv_producer and not self.is_kv_consumer: with_prefill = True + has_lora = True if self.lora_config and self.compilation_config.cudagraph_specialize_lora else False + _ag_mode, batch_descriptor = self.cudagraph_dispatcher.dispatch( + num_tokens=num_tokens, uniform_decode=uniform_decode, has_lora=has_lora + ) + # Padding for DP - (num_tokens, num_tokens_across_dp, with_prefill) = self._sync_metadata_across_dp(num_tokens, with_prefill) + (num_tokens, num_tokens_across_dp, with_prefill) = self._sync_metadata_across_dp( + batch_descriptor.num_tokens, with_prefill + ) # If cudagraph_mode.decode_mode() == FULL and # cudagraph_mode.separate_routine(). This means that we are using @@ -830,13 +385,13 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): num_scheduled_tokens = np.array(num_scheduled_tokens_list, dtype=np.int32) num_sampled_tokens = np.ones(num_reqs, dtype=np.int32) - if not self.in_profile_run and self.dynamic_eplb: + if not is_profile and self.dynamic_eplb: self.eplb_updator.forward_before() - has_lora = True if self.lora_config and self.compilation_config.cudagraph_specialize_lora else False - _ag_mode, batch_descriptor = self.cudagraph_dispatcher.dispatch( - num_tokens=num_tokens, uniform_decode=uniform_decode, has_lora=has_lora - ) + if num_tokens != batch_descriptor.num_tokens: + _ag_mode, batch_descriptor = self.cudagraph_dispatcher.dispatch( + num_tokens=num_tokens, uniform_decode=uniform_decode, has_lora=has_lora + ) num_tokens_padded = batch_descriptor.num_tokens num_reqs_padded = batch_descriptor.num_reqs if batch_descriptor.num_reqs is not None else num_reqs @@ -845,19 +400,17 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): num_tokens_across_dp[:] = num_tokens_padded num_scheduled_tokens = num_scheduled_tokens.repeat(num_reqs_padded) - moe_comm_type = self._select_moe_comm_method(num_tokens_padded) - # filter out the valid batch descriptor - if aclgraph_runtime_mode is not None: + if cudagraph_runtime_mode is not None: # we allow forcing NONE when the dispatcher disagrees to support # warm ups for aclgraph capture - if aclgraph_runtime_mode != CUDAGraphMode.NONE and aclgraph_runtime_mode != _ag_mode: + if cudagraph_runtime_mode != CUDAGraphMode.NONE and cudagraph_runtime_mode != _ag_mode: raise ValueError( f"Aclgraph runtime mode mismatch at dummy_run. " - f"Expected {_ag_mode}, but got {aclgraph_runtime_mode}." + f"Expected {_ag_mode}, but got {cudagraph_runtime_mode}." ) else: - aclgraph_runtime_mode = _ag_mode + cudagraph_runtime_mode = _ag_mode # TODO(Mengqing): Set create_mixed_batch to False since it's only used in FI warmup # and not supported in ASCEND now. We could remove it in the future. @@ -866,15 +419,16 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): num_reqs=num_reqs_padded, num_tokens=num_tokens_padded, max_query_len=max_query_len, - aclgraph_runtime_mode=aclgraph_runtime_mode, + aclgraph_runtime_mode=cudagraph_runtime_mode, force_attention=force_attention, + is_graph_capturing=is_graph_capturing, num_scheduled_tokens=num_scheduled_tokens, ) with self.maybe_dummy_run_with_lora(self.lora_config, num_scheduled_tokens, num_sampled_tokens): # Make sure padding doesn't exceed max_num_tokens assert num_tokens_padded <= self.max_num_tokens - if self.is_multimodal_model: + if self.is_multimodal_model and not self.model_config.is_encoder_decoder: input_ids = None inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded] elif self.enable_prompt_embeds: @@ -886,9 +440,14 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): if self.uses_mrope: positions = self.mrope_positions.gpu[:, :num_tokens_padded] + elif self.uses_xdrope_dim > 0: + positions = self.xdrope_positions.gpu[:, :num_tokens_padded] else: positions = self.positions.gpu[:num_tokens_padded] + # update global cos, sin + update_cos_sin(positions) + if get_pp_group().is_first_rank: intermediate_tensors = None else: @@ -908,7 +467,7 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): {k: v[:num_tokens_padded] for k, v in self.intermediate_tensors.items()} ) - need_dummy_logits = not self.in_profile_run and lmhead_tp_enable() + need_dummy_logits = not is_profile and lmhead_tp_enable() max_num_reqs_across_dp = max_num_reqs * self.uniform_decode_query_len dummy_indices = torch.zeros(max_num_reqs_across_dp, dtype=torch.int32) @@ -928,16 +487,11 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): self.vllm_config, num_tokens=num_tokens_padded, num_tokens_across_dp=num_tokens_across_dp, - with_prefill=with_prefill, - in_profile_run=self.in_profile_run, - # reserved_mc2_mask=self.reserved_mc2_mask, - moe_comm_type=moe_comm_type, + in_profile_run=is_profile, num_actual_tokens=0, - aclgraph_runtime_mode=aclgraph_runtime_mode, + aclgraph_runtime_mode=cudagraph_runtime_mode, batch_descriptor=batch_descriptor, - prefetch_stream=self.prefetch_stream, model_instance=self.model, - weight_prefetch_method=self.weight_prefetch_method, ): hidden_states = self._generate_dummy_run_hidden_states( input_ids, positions, num_tokens_padded, intermediate_tensors, inputs_embeds @@ -950,14 +504,15 @@ class NPUGenerationModelRunner(OmniNPUModelRunner): with_prefill=with_prefill, num_reqs=num_reqs_padded, num_tokens_across_dp=num_tokens_across_dp, - aclgraph_runtime_mode=aclgraph_runtime_mode, + aclgraph_runtime_mode=cudagraph_runtime_mode, batch_descriptor=batch_descriptor, dummy_compute_logits=dummy_drafter_compute_logits, - skip_attn=not force_attention, + in_graph_capturing=not force_attention, + is_profile=is_profile, ) - if self.in_profile_run and self.dynamic_eplb: + if is_profile and self.dynamic_eplb: self.model.clear_all_moe_loads() - if not self.in_profile_run and self.dynamic_eplb: + if self.dynamic_eplb: self.eplb_updator.take_update_info_from_eplb_process() self.eplb_updator.forward_end() # -------------------------------------- Omni-new ------------------------------------------------- diff --git a/vllm_omni/worker/npu/npu_generation_worker.py b/vllm_omni/worker/npu/npu_generation_worker.py index 23cdf41c7..588d21fc1 100644 --- a/vllm_omni/worker/npu/npu_generation_worker.py +++ b/vllm_omni/worker/npu/npu_generation_worker.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm_ascend.worker.worker_v1 import NPUWorker +from vllm.v1.worker.workspace import init_workspace_manager +from vllm_ascend.worker.worker import NPUWorker from vllm_omni.worker.npu.npu_generation_model_runner import NPUGenerationModelRunner @@ -10,6 +11,8 @@ class NPUGenerationWorker(NPUWorker): """NPU generation worker for code2wav stage in Omni model.""" def init_device(self): - device = self._init_device() + self.device = self._init_device() + num_ubatches = 1 + init_workspace_manager(self.device, num_ubatches) - self.model_runner: NPUGenerationModelRunner = NPUGenerationModelRunner(self.vllm_config, device) + self.model_runner = NPUGenerationModelRunner(self.vllm_config, self.device) diff --git a/vllm_omni/worker/npu/npu_model_runner.py b/vllm_omni/worker/npu/npu_model_runner.py index 264d5e341..b0fbf7a4e 100644 --- a/vllm_omni/worker/npu/npu_model_runner.py +++ b/vllm_omni/worker/npu/npu_model_runner.py @@ -18,6 +18,8 @@ from vllm.sequence import IntermediateTensors from vllm.utils.math_utils import cdiv from vllm.v1.worker.gpu_input_batch import CachedRequestState from vllm_ascend.ascend_forward_context import set_ascend_forward_context +from vllm_ascend.compilation.acl_graph import ACLGraphWrapper +from vllm_ascend.ops.rotary_embedding import update_cos_sin from vllm_ascend.utils import enable_sp, lmhead_tp_enable from vllm_ascend.worker.model_runner_v1 import NPUModelRunner @@ -40,6 +42,25 @@ class OmniNPUModelRunner(NPUModelRunner): self._omni_num_scheduled_tokens_np: np.ndarray | None = None self._omni_last_model_output: object | None = None + def load_model(self, *args, **kwargs) -> None: + super().load_model(*args, **kwargs) + # TODO move this model specific logic to a separate class + if hasattr(self.model, "talker_mtp") and self.model.talker is not None: + self.talker_mtp = self.model.talker_mtp + cudagraph_mode = self.compilation_config.cudagraph_mode + assert cudagraph_mode is not None + if cudagraph_mode.has_full_cudagraphs(): + self.talker_mtp = ACLGraphWrapper( + self.model.talker_mtp, self.vllm_config, runtime_mode=CUDAGraphMode.FULL + ) + hidden_size = self.model_config.hf_config.talker_config.text_config.hidden_size + self.talker_mtp_input_ids = self._make_buffer(self.max_num_reqs, dtype=torch.int32) + self.talker_mtp_inputs_embeds = self._make_buffer( + self.max_num_reqs, hidden_size, dtype=self.dtype, numpy=False + ) + self.last_talker_hidden = self._make_buffer(self.max_num_reqs, hidden_size, dtype=self.dtype, numpy=False) + self.text_step = self._make_buffer(self.max_num_reqs, hidden_size, dtype=self.dtype, numpy=False) + def _init_mrope_positions(self, req_state: CachedRequestState): image_grid_thw = [] video_grid_thw = [] @@ -99,6 +120,7 @@ class OmniNPUModelRunner(NPUModelRunner): # Remove finished requests from the cached states. for req_id in scheduler_output.finished_req_ids: self.requests.pop(req_id, None) + self.num_prompt_logprobs.pop(req_id, None) # Remove the finished requests from the persistent batch. # NOTE(woosuk): There could be an edge case where finished_req_ids and # scheduled_req_ids overlap. This happens when a request is aborted and @@ -119,7 +141,14 @@ class OmniNPUModelRunner(NPUModelRunner): # they will be scheduled again sometime in the future. scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys() cached_req_ids = self.input_batch.req_id_to_index.keys() - unscheduled_req_ids = cached_req_ids - scheduled_req_ids + resumed_req_ids = scheduler_output.scheduled_cached_reqs.resumed_req_ids + # NOTE(zhuohan): cached_req_ids and resumed_req_ids are usually disjoint, + # so `(scheduled_req_ids - resumed_req_ids) == scheduled_req_ids` holds + # apart from the forced-preemption case in reset_prefix_cache. And in + # that case we include the resumed_req_ids in the unscheduled set so + # that they get cleared from the persistent batch before being re-scheduled + # in the normal resumed request path. + unscheduled_req_ids = cached_req_ids - (scheduled_req_ids - resumed_req_ids) # NOTE(woosuk): The persistent batch optimization assumes that # consecutive batches contain mostly the same requests. If batches # have low request overlap (e.g., alternating between two distinct @@ -211,21 +240,64 @@ class OmniNPUModelRunner(NPUModelRunner): except Exception as e: logger.error(f"Error decoding additional information: {e}") pass - # ------------------------------------------------------------------------------------------------ + # -------------------------------------- Omni-new ------------------------------------------------- + + if sampling_params and sampling_params.prompt_logprobs is not None: + self.num_prompt_logprobs[req_id] = ( + self.input_batch.vocab_size + if sampling_params.prompt_logprobs == -1 + else sampling_params.prompt_logprobs + ) # Only relevant for models using M-RoPE (e.g, Qwen2-VL) if self.uses_mrope: self._init_mrope_positions(req_state) + # Only relevant for models using XD-RoPE (e.g, HunYuan-VL) + if self.uses_xdrope_dim > 0: + self._init_xdrope_positions(req_state) + reqs_to_add.append(self.requests[req_id]) # Update the states of the running/resumed requests. is_last_rank = get_pp_group().is_last_rank req_data = scheduler_output.scheduled_cached_reqs + scheduled_spec_tokens = scheduler_output.scheduled_spec_decode_tokens + + # Wait until valid_sampled_tokens_count is copied to cpu, + # then use it to update actual num_computed_tokens of each request. + valid_sampled_token_count = self._get_valid_sampled_token_count() + for i, req_id in enumerate(req_data.req_ids): req_state = self.requests[req_id] num_computed_tokens = req_data.num_computed_tokens[i] new_block_ids = req_data.new_block_ids[i] - resumed_from_preemption = req_data.resumed_from_preemption[i] + resumed_from_preemption = req_id in req_data.resumed_req_ids + num_output_tokens = req_data.num_output_tokens[i] + req_index = self.input_batch.req_id_to_index.get(req_id) + + if req_state.prev_num_draft_len and self.use_async_scheduling: + # prev_num_draft_len is used in async scheduling mode with + # spec decode. it indicates if need to update num_computed_tokens + # of the request. for example: + # fist step: num_computed_tokens = 0, spec_tokens = [], + # prev_num_draft_len = 0. + # second step: num_computed_tokens = 100(prompt length), + # spec_tokens = [a,b], prev_num_draft_len = 0. + # third step: num_computed_tokens = 100 + 2, spec_tokens = [c,d], + # prev_num_draft_len = 2. + # num_computed_tokens in first step and second step does't contain + # the spec tokens length, but in third step it contains the + # spec tokens length. we only need to update num_computed_tokens + # when prev_num_draft_len > 0. + if req_index is None: + req_state.prev_num_draft_len = 0 + else: + assert self.input_batch.prev_req_id_to_index is not None + prev_req_index = self.input_batch.prev_req_id_to_index[req_id] + num_accepted = valid_sampled_token_count[prev_req_index] - 1 + num_rejected = req_state.prev_num_draft_len - num_accepted + num_computed_tokens -= num_rejected + req_state.output_token_ids.extend([-1] * num_accepted) # Update the cached states. req_state.num_computed_tokens = num_computed_tokens @@ -243,6 +315,13 @@ class OmniNPUModelRunner(NPUModelRunner): req_state.output_token_ids.append(new_token_ids[-1]) elif num_new_tokens > 0: req_state.output_token_ids.extend(new_token_ids[-num_new_tokens:]) + elif num_output_tokens < len(req_state.output_token_ids): + # Some output tokens were discarded due to a sync-KV-load + # failure. Align the cached state. + del req_state.output_token_ids[num_output_tokens:] + if req_index is not None: + end_idx = self.input_batch.num_prompt_tokens[req_index] + num_output_tokens + self.input_batch.num_tokens_no_spec[req_index] = end_idx # Update the block IDs. if not resumed_from_preemption: @@ -251,6 +330,7 @@ class OmniNPUModelRunner(NPUModelRunner): for block_ids, new_ids in zip(req_state.block_ids, new_block_ids): block_ids.extend(new_ids) else: + assert req_index is None assert new_block_ids is not None # The request is resumed from preemption. # Replace the existing block IDs with the new ones. @@ -261,6 +341,13 @@ class OmniNPUModelRunner(NPUModelRunner): # The request is not in the persistent batch. # The request was either preempted and resumed later, or was not # scheduled in the previous step and needs to be added again. + + if self.use_async_scheduling and num_output_tokens > 0: + # We must recover the output token ids for resumed requests in the + # async scheduling case, so that correct input_ids are obtained. + resumed_token_ids = req_data.all_token_ids[req_id] + req_state.output_token_ids = resumed_token_ids[-num_output_tokens:] + reqs_to_add.append(req_state) continue @@ -277,22 +364,15 @@ class OmniNPUModelRunner(NPUModelRunner): end_token_index = num_computed_tokens + len(new_token_ids) self.input_batch.token_ids_cpu[req_index, start_token_index:end_token_index] = new_token_ids self.input_batch.num_tokens_no_spec[req_index] = end_token_index - self.input_batch.num_tokens[req_index] = end_token_index # Add spec_token_ids to token_ids_cpu. - spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id, ()) - if spec_token_ids: - num_spec_tokens = len(spec_token_ids) - start_index = self.input_batch.num_tokens_no_spec[req_index] - end_token_index = start_index + num_spec_tokens - self.input_batch.token_ids_cpu[req_index, start_index:end_token_index] = spec_token_ids - # NOTE(woosuk): `num_tokens` here may include spec tokens. - self.input_batch.num_tokens[req_index] += num_spec_tokens + self.input_batch.update_req_spec_token_ids(req_state, scheduled_spec_tokens) # Add the new or resumed requests to the persistent batch. # The smaller empty indices are filled first. for request in reqs_to_add: self.input_batch.add_request(request) + self.input_batch.update_req_spec_token_ids(request, scheduled_spec_tokens) # Condense the batched states if there are gaps left by removed requests self.input_batch.condense() @@ -302,18 +382,19 @@ class OmniNPUModelRunner(NPUModelRunner): self.input_batch.refresh_metadata() @torch.inference_mode() - def extract_multimodal_outputs( - self, hidden_states: torch.Tensor | list[torch.Tensor] | OmniOutput - ) -> tuple[torch.Tensor, torch.Tensor | list[torch.Tensor] | dict]: - """Extract multimodal outputs from hidden states.""" - if hasattr(self.model, "have_multimodal_outputs") and self.model.have_multimodal_outputs: + def extract_multimodal_outputs(self, hidden_states: torch.Tensor | list[torch.Tensor] | OmniOutput) -> dict: + if ( + hasattr(self.model, "have_multimodal_outputs") + and self.model.have_multimodal_outputs + and isinstance(hidden_states, OmniOutput) + ): text_hidden_states = hidden_states.text_hidden_states multimodal_outputs = hidden_states.multimodal_outputs elif isinstance(hidden_states, torch.Tensor): text_hidden_states = hidden_states multimodal_outputs = {} - elif isinstance(hidden_states, list): + elif isinstance(hidden_states, list) or isinstance(hidden_states, tuple): text_hidden_states = hidden_states[0] multimodal_outputs = {} else: @@ -325,12 +406,18 @@ class OmniNPUModelRunner(NPUModelRunner): self, num_tokens: int, with_prefill: bool = False, - aclgraph_runtime_mode: CUDAGraphMode | None = None, + cudagraph_runtime_mode: CUDAGraphMode | None = None, force_attention: bool = False, uniform_decode: bool = False, + is_profile: bool = False, + allow_microbatching: bool = True, + skip_eplb: bool = False, + remove_lora: bool = True, + activate_lora: bool = False, + is_graph_capturing: bool = False, ) -> torch.Tensor: # only support eager mode and piecewise graph now - assert aclgraph_runtime_mode is None or aclgraph_runtime_mode in { + assert cudagraph_runtime_mode is None or cudagraph_runtime_mode in { CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL, @@ -345,8 +432,15 @@ class OmniNPUModelRunner(NPUModelRunner): if self.is_kv_producer and not self.is_kv_consumer: with_prefill = True + has_lora = True if self.lora_config and self.compilation_config.cudagraph_specialize_lora else False + _ag_mode, batch_descriptor = self.cudagraph_dispatcher.dispatch( + num_tokens=num_tokens, uniform_decode=uniform_decode, has_lora=has_lora + ) + # Padding for DP - (num_tokens, num_tokens_across_dp, with_prefill) = self._sync_metadata_across_dp(num_tokens, with_prefill) + (num_tokens, num_tokens_across_dp, with_prefill) = self._sync_metadata_across_dp( + batch_descriptor.num_tokens, with_prefill + ) # If cudagraph_mode.decode_mode() == FULL and # cudagraph_mode.separate_routine(). This means that we are using @@ -387,13 +481,13 @@ class OmniNPUModelRunner(NPUModelRunner): num_scheduled_tokens = np.array(num_scheduled_tokens_list, dtype=np.int32) num_sampled_tokens = np.ones(num_reqs, dtype=np.int32) - if not self.in_profile_run and self.dynamic_eplb: + if not is_profile and self.dynamic_eplb: self.eplb_updator.forward_before() - has_lora = True if self.lora_config and self.compilation_config.cudagraph_specialize_lora else False - _ag_mode, batch_descriptor = self.cudagraph_dispatcher.dispatch( - num_tokens=num_tokens, uniform_decode=uniform_decode, has_lora=has_lora - ) + if num_tokens != batch_descriptor.num_tokens: + _ag_mode, batch_descriptor = self.cudagraph_dispatcher.dispatch( + num_tokens=num_tokens, uniform_decode=uniform_decode, has_lora=has_lora + ) num_tokens_padded = batch_descriptor.num_tokens num_reqs_padded = batch_descriptor.num_reqs if batch_descriptor.num_reqs is not None else num_reqs @@ -402,19 +496,17 @@ class OmniNPUModelRunner(NPUModelRunner): num_tokens_across_dp[:] = num_tokens_padded num_scheduled_tokens = num_scheduled_tokens.repeat(num_reqs_padded) - moe_comm_type = self._select_moe_comm_method(num_tokens_padded) - # filter out the valid batch descriptor - if aclgraph_runtime_mode is not None: + if cudagraph_runtime_mode is not None: # we allow forcing NONE when the dispatcher disagrees to support # warm ups for aclgraph capture - if aclgraph_runtime_mode != CUDAGraphMode.NONE and aclgraph_runtime_mode != _ag_mode: + if cudagraph_runtime_mode != CUDAGraphMode.NONE and cudagraph_runtime_mode != _ag_mode: raise ValueError( f"Aclgraph runtime mode mismatch at dummy_run. " - f"Expected {_ag_mode}, but got {aclgraph_runtime_mode}." + f"Expected {_ag_mode}, but got {cudagraph_runtime_mode}." ) else: - aclgraph_runtime_mode = _ag_mode + cudagraph_runtime_mode = _ag_mode # TODO(Mengqing): Set create_mixed_batch to False since it's only used in FI warmup # and not supported in ASCEND now. We could remove it in the future. @@ -423,15 +515,16 @@ class OmniNPUModelRunner(NPUModelRunner): num_reqs=num_reqs_padded, num_tokens=num_tokens_padded, max_query_len=max_query_len, - aclgraph_runtime_mode=aclgraph_runtime_mode, + aclgraph_runtime_mode=cudagraph_runtime_mode, force_attention=force_attention, + is_graph_capturing=is_graph_capturing, num_scheduled_tokens=num_scheduled_tokens, ) with self.maybe_dummy_run_with_lora(self.lora_config, num_scheduled_tokens, num_sampled_tokens): # Make sure padding doesn't exceed max_num_tokens assert num_tokens_padded <= self.max_num_tokens - if self.is_multimodal_model: + if self.is_multimodal_model and not self.model_config.is_encoder_decoder: input_ids = None inputs_embeds = self.inputs_embeds.gpu[:num_tokens_padded] elif self.enable_prompt_embeds: @@ -443,9 +536,14 @@ class OmniNPUModelRunner(NPUModelRunner): if self.uses_mrope: positions = self.mrope_positions.gpu[:, :num_tokens_padded] + elif self.uses_xdrope_dim > 0: + positions = self.xdrope_positions.gpu[:, :num_tokens_padded] else: positions = self.positions.gpu[:num_tokens_padded] + # update global cos, sin + update_cos_sin(positions) + if get_pp_group().is_first_rank: intermediate_tensors = None else: @@ -465,7 +563,7 @@ class OmniNPUModelRunner(NPUModelRunner): {k: v[:num_tokens_padded] for k, v in self.intermediate_tensors.items()} ) - need_dummy_logits = not self.in_profile_run and lmhead_tp_enable() + need_dummy_logits = not is_profile and lmhead_tp_enable() max_num_reqs_across_dp = max_num_reqs * self.uniform_decode_query_len dummy_indices = torch.zeros(max_num_reqs_across_dp, dtype=torch.int32) @@ -485,42 +583,44 @@ class OmniNPUModelRunner(NPUModelRunner): self.vllm_config, num_tokens=num_tokens_padded, num_tokens_across_dp=num_tokens_across_dp, - with_prefill=with_prefill, - in_profile_run=self.in_profile_run, - # reserved_mc2_mask=self.reserved_mc2_mask, - moe_comm_type=moe_comm_type, + in_profile_run=is_profile, num_actual_tokens=0, - aclgraph_runtime_mode=aclgraph_runtime_mode, + aclgraph_runtime_mode=cudagraph_runtime_mode, batch_descriptor=batch_descriptor, - prefetch_stream=self.prefetch_stream, model_instance=self.model, - weight_prefetch_method=self.weight_prefetch_method, ): + if getattr(self.model, "talker", None) is not None and hasattr(self.model, "talker_mtp"): + hidden_states = self.talker_mtp( + self.talker_mtp_input_ids.gpu[:num_tokens_padded], + self.talker_mtp_inputs_embeds.gpu[:num_tokens_padded], + self.last_talker_hidden.gpu[:num_tokens_padded], + self.text_step.gpu[:num_tokens_padded], + ) hidden_states = self._generate_dummy_run_hidden_states( input_ids, positions, num_tokens_padded, intermediate_tensors, inputs_embeds ) dummy_compute_logits(hidden_states) + hidden_states, _ = self.extract_multimodal_outputs(hidden_states) + if self.drafter: self.drafter.dummy_run( num_tokens=num_tokens_padded, with_prefill=with_prefill, num_reqs=num_reqs_padded, num_tokens_across_dp=num_tokens_across_dp, - aclgraph_runtime_mode=aclgraph_runtime_mode, + aclgraph_runtime_mode=cudagraph_runtime_mode, batch_descriptor=batch_descriptor, dummy_compute_logits=dummy_drafter_compute_logits, - skip_attn=not force_attention, + in_graph_capturing=not force_attention, + is_profile=is_profile, ) - if self.in_profile_run and self.dynamic_eplb: + if is_profile and self.dynamic_eplb: self.model.clear_all_moe_loads() - if not self.in_profile_run and self.dynamic_eplb: + if self.dynamic_eplb: self.eplb_updator.take_update_info_from_eplb_process() self.eplb_updator.forward_end() - - hidden_states, _ = self.extract_multimodal_outputs(hidden_states) - - return hidden_states + return hidden_states, hidden_states def _decode_and_store_request_payloads(self, scheduler_output: "SchedulerOutput") -> None: """Decode per-request prompt_embeds and additional_information for newly @@ -615,6 +715,43 @@ class OmniNPUModelRunner(NPUModelRunner): traceback.print_exc() return model_kwargs_extra + def _process_additional_information_updates( + self, + hidden_states: torch.Tensor, + multimodal_outputs: object, + num_scheduled_tokens_np: np.ndarray, + scheduler_output: "SchedulerOutput", + ) -> None: + """Process model-provided per-request additional_information updates and merge into request state.""" + try: + # execute the custom postprocess function + # TODO(Peiqi): do we have a more elegant way to do this? + if hasattr(self.model, "has_postprocess") and self.model.has_postprocess: + for req_index, req_id in enumerate(self.input_batch.req_ids): + if self.model_config.async_chunk: + req_infos = self._get_additional_information(scheduler_output, req_id) + else: + req_state = self.requests.get(req_id) + req_infos = ( + getattr(req_state, "additional_information_cpu", None) if req_state is not None else None + ) + start_offset = int(self.query_start_loc.cpu[req_index]) + sched_tokens = int(num_scheduled_tokens_np[req_index]) + s, e = start_offset, start_offset + sched_tokens + # only consider to store data into update dict. + hidden_states_slice = hidden_states[s:e] + update_dict = self.model.postprocess(hidden_states_slice, **req_infos) + self._merge_additional_information_update(req_id, update_dict) + except Exception as e: + logger.error( + f"Error merging for requests:{self.input_batch.req_ids} " + f"additional information update: {e}, with the multimodal_outputs " + f"as {multimodal_outputs}" + ) + import traceback + + traceback.print_exc() + def _collect_additional_information_for_prefill( self, num_scheduled_tokens_np: np.ndarray, @@ -638,6 +775,231 @@ class OmniNPUModelRunner(NPUModelRunner): start_offset = int(self.query_start_loc.cpu[req_index]) self.inputs_embeds[start_offset : start_offset + overlay_len].copy_(src) + def _get_additional_information(self, scheduler_output: "SchedulerOutput", req_id: str) -> dict: + req_infos = None + req_state = self.requests.get(req_id) + additional_information_cpu = getattr(req_state, "additional_information_cpu", None) + for new_req in scheduler_output.scheduled_new_reqs: + if new_req.req_id == req_id: + payload_info = getattr(new_req, "additional_information", None) + if payload_info is not None: + return payload_info + + if hasattr(scheduler_output.scheduled_cached_reqs, "additional_information"): + cached_infos = getattr(scheduler_output.scheduled_cached_reqs, "additional_information", {}) + if isinstance(cached_infos, dict) and req_id in cached_infos: + req_infos = cached_infos[req_id] + if not isinstance(req_infos, dict): + req_infos = None + + if req_infos is None or req_infos.get("last_talker_hidden", None) is None: + if req_infos is None: + additional_information_cpu.pop("thinker_embeddings", None) + req_infos = additional_information_cpu + else: + req_infos["last_talker_hidden"] = additional_information_cpu.get("last_talker_hidden", None) + req_infos["num_processed_thinker_tokens"] = additional_information_cpu.get( + "num_processed_thinker_tokens", 0 + ) + if not isinstance(req_infos, dict): + req_infos = None + + if req_infos is None: + logger.warning(f"No additional_information found for req_id: {req_id}") + + return req_infos + + def _preprocess( + self, + scheduler_output: "SchedulerOutput", + num_input_tokens: int, + intermediate_tensors: IntermediateTensors | None = None, + ): + """Align with v0.14.0 preprocess and omni's additional information handling.""" + num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens + is_first_rank = get_pp_group().is_first_rank + is_encoder_decoder = self.model_config.is_encoder_decoder + + # _prepare_inputs may reorder the batch, so we must gather multi + # modal outputs after that to ensure the correct order + ec_connector_output = None + + if self.supports_mm_inputs and is_first_rank and not is_encoder_decoder: + # Run the multimodal encoder if any. + with self.maybe_get_ec_connector_output( + scheduler_output, + encoder_cache=self.encoder_cache, + ) as ec_connector_output: + self._execute_mm_encoder(scheduler_output) + mm_embeds, is_mm_embed = self._gather_mm_embeddings(scheduler_output) + + # NOTE(woosuk): To unify token ids and soft tokens (vision + # embeddings), we always use embeddings (rather than token ids) + # as input to the multimodal model, even when the input is text. + inputs_embeds_scheduled = self.model.embed_input_ids( + self.input_ids.gpu[:num_scheduled_tokens], + multimodal_embeddings=mm_embeds, + is_multimodal=is_mm_embed, + ) + + # TODO(woosuk): Avoid the copy. Optimize. + self.inputs_embeds.gpu[:num_scheduled_tokens].copy_(inputs_embeds_scheduled) + + input_ids, inputs_embeds = self._prepare_mm_inputs(num_input_tokens) + model_kwargs = { + **self._init_model_kwargs(), + **self._extract_mm_kwargs(scheduler_output), + } + elif self.enable_prompt_embeds and is_first_rank: + # Get the input embeddings for the tokens that are not input embeds, + # then put them into the appropriate positions. + # TODO(qthequartermasterman): Since even when prompt embeds are + # enabled, (a) not all requests will use prompt embeds, and (b) + # after the initial prompt is processed, the rest of the generated + # tokens will be token ids, it is not desirable to have the + # embedding layer outside of the CUDA graph all the time. The v0 + # engine avoids this by "double compiling" the CUDA graph, once + # with input_ids and again with inputs_embeds, for all num_tokens. + # If a batch only has token ids, then including the embedding layer + # in the CUDA graph will be more performant (like in the else case + # below). + token_ids_idx = self.is_token_ids.gpu[:num_scheduled_tokens].nonzero(as_tuple=False).squeeze(1) + # Some tokens ids may need to become embeds + if token_ids_idx.numel() > 0: + token_ids = self.input_ids.gpu[token_ids_idx] + tokens_to_embeds = self.model.embed_input_ids(input_ids=token_ids) + self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds + + inputs_embeds = self.inputs_embeds.gpu[:num_input_tokens] + model_kwargs = self._init_model_kwargs() + input_ids = self.input_ids.gpu[:num_input_tokens] + else: + # For text-only models, we use token ids as input. + # While it is possible to use embeddings as input just like the + # multimodal models, it is not desirable for performance since + # then the embedding layer is not included in the CUDA graph. + input_ids = self.input_ids.gpu[:num_input_tokens] + inputs_embeds = None + model_kwargs = self._init_model_kwargs() + + if self.uses_mrope: + positions = self.mrope_positions.gpu[:, :num_input_tokens] + elif self.uses_xdrope_dim > 0: + positions = self.xdrope_positions.gpu[:, :num_input_tokens] + else: + positions = self.positions.gpu[:num_input_tokens] + + if is_first_rank: + intermediate_tensors = None + else: + assert intermediate_tensors is not None + intermediate_tensors = self.sync_and_slice_intermediate_tensors( + num_input_tokens, intermediate_tensors, True + ) + + if is_encoder_decoder and scheduler_output.scheduled_encoder_inputs: + # Run the encoder, just like we do with other multimodal inputs. + # For an encoder-decoder model, our processing here is a bit + # simpler, because the outputs are just passed to the decoder. + # We are not doing any prompt replacement. We also will only + # ever have a single encoder input. + encoder_outputs = self._execute_mm_encoder(scheduler_output) + model_kwargs.update({"encoder_outputs": encoder_outputs}) + + req_ids = self.input_batch.req_ids + num_scheduled_tokens_np = np.array( + [scheduler_output.num_scheduled_tokens[rid] for rid in req_ids], + dtype=np.int32, + ) + self._omni_num_scheduled_tokens_np = num_scheduled_tokens_np + + # Note: only prefill need collect additional_information for now. + # Decode don't need per_req_additional_information anymore. + if inputs_embeds is not None: + # Prefill: overlay prompt_embeds and collect additional_information + self._collect_additional_information_for_prefill(num_scheduled_tokens_np) + + if hasattr(self.model, "has_preprocess") and self.model.has_preprocess: + # Overlay custom prompt_embeds per request for the prompt portion; + # collect additional_information (tensor/list) for prefill portion only + decode_req_ids = [] + for req_index, req_id in enumerate(self.input_batch.req_ids): + # Try to get additional_information from multiple sources + if self.vllm_config.model_config.async_chunk: + req_infos = self._get_additional_information(scheduler_output, req_id) + else: + req_state = self.requests.get(req_id) + req_infos = ( + getattr(req_state, "additional_information_cpu", None) if req_state is not None else None + ) + start_offset = int(self.query_start_loc.cpu[req_index]) + sched_tokens = int(num_scheduled_tokens_np[req_index]) + s, e = start_offset, start_offset + sched_tokens + span_len = int(e) - int(s) + + # call the custom process function + req_input_ids, req_embeds, update_dict = self.model.preprocess( + input_ids=input_ids[s:e], input_embeds=inputs_embeds[s:e], **req_infos + ) + if hasattr(self.model, "talker_mtp") and span_len == 1: + last_talker_hidden, text_step = update_dict.pop("mtp_inputs") + decode_slice = slice(len(decode_req_ids), len(decode_req_ids) + 1) + self.talker_mtp_input_ids.gpu[decode_slice].copy_(req_input_ids) + self.talker_mtp_inputs_embeds.gpu[decode_slice].copy_(req_embeds) + self.last_talker_hidden.gpu[decode_slice].copy_(last_talker_hidden) + self.text_step.gpu[decode_slice].copy_(text_step) + decode_req_ids.append(req_id) + + # TODO(Peiqi): the merge stage could move out from the critical path + self._merge_additional_information_update(req_id, update_dict) + + # update the inputs_embeds and input_ids + seg_len = min(span_len, req_embeds.shape[0]) + inputs_embeds[s : s + seg_len] = req_embeds[:seg_len] + if isinstance(req_input_ids, torch.Tensor) and req_input_ids.numel() == seg_len: + input_ids[s : s + seg_len] = req_input_ids + + # run talker mtp decode + if hasattr(self.model, "talker_mtp"): + self._talker_mtp_forward(decode_req_ids, inputs_embeds) + + return ( + input_ids, + inputs_embeds, + positions, + intermediate_tensors, + model_kwargs, + ec_connector_output, + ) + + def _talker_mtp_forward(self, decode_req_ids: list[str], inputs_embeds: torch.Tensor) -> None: + decode_batch_size = len(decode_req_ids) + if decode_batch_size == 0: + return + _cudagraph_mode, batch_desc, _, _, _ = self._determine_batch_execution_and_padding( + num_tokens=decode_batch_size, + num_reqs=decode_batch_size, + num_scheduled_tokens_np=np.ones(decode_batch_size, dtype=np.int32), + max_num_scheduled_tokens=1, + use_cascade_attn=False, + ) + req_input_ids = self.talker_mtp_input_ids.gpu[:decode_batch_size] + req_embeds = self.talker_mtp_inputs_embeds.gpu[:decode_batch_size] + last_talker_hidden = self.last_talker_hidden.gpu[:decode_batch_size] + text_step = self.text_step.gpu[:decode_batch_size] + with set_ascend_forward_context( + None, self.vllm_config, aclgraph_runtime_mode=_cudagraph_mode, batch_descriptor=batch_desc + ): + req_embeds, code_predictor_codes = self.talker_mtp(req_input_ids, req_embeds, last_talker_hidden, text_step) + # update the inputs_embeds and code_predictor_codes + code_predictor_codes_cpu = code_predictor_codes.detach().to("cpu").contiguous() + for idx, req_id in enumerate(decode_req_ids): + req_index = self.input_batch.req_ids.index(req_id) + start_offset = int(self.query_start_loc.cpu[req_index]) + inputs_embeds[start_offset : start_offset + 1] = req_embeds[idx : idx + 1] + update_dict = {"code_predictor_codes": code_predictor_codes_cpu[idx : idx + 1]} + self._merge_additional_information_update(req_id, update_dict) + def _model_forward( self, input_ids: torch.Tensor | None = None, @@ -663,6 +1025,27 @@ class OmniNPUModelRunner(NPUModelRunner): **model_kwargs, **model_kwargs_extra, ) + if not isinstance(model_output, OmniOutput) and hasattr(self.model, "make_omni_output"): + model_output = self.model.make_omni_output(model_output, **model_kwargs_extra) # Cache model output so later sample_tokens can consume multimodal results. self._omni_last_model_output = model_output return model_output + + def _merge_additional_information_update(self, req_id: str, upd: dict) -> None: + req_state = self.requests.get(req_id) + if req_state is None: + return + existing = getattr(req_state, "additional_information_cpu", {}) + if not isinstance(existing, dict): + existing = {} + merged = dict(existing) + for k, v in upd.items(): + if isinstance(v, torch.Tensor): + merged[k] = v.detach().to("cpu").contiguous() + elif isinstance(v, list): + merged[k] = [ + (item.detach().to("cpu").contiguous() if isinstance(item, torch.Tensor) else item) for item in v + ] + else: + merged[k] = v + setattr(req_state, "additional_information_cpu", merged) From 46e96e18e37b97b596de36f04d5c8104a801e8ce Mon Sep 17 00:00:00 2001 From: TJian Date: Tue, 27 Jan 2026 20:14:49 +0800 Subject: [PATCH 27/83] [ROCm] [Perf] Add AITER Flash Attention (#941) Signed-off-by: tjtanaa --- .../attention/backends/flash_attn.py | 9 ++++++-- .../diffusion/attention/backends/utils/fa.py | 14 ++++++++++++ vllm_omni/diffusion/attention/selector.py | 22 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/vllm_omni/diffusion/attention/backends/flash_attn.py b/vllm_omni/diffusion/attention/backends/flash_attn.py index 7fad4ce26..e9102b1d3 100644 --- a/vllm_omni/diffusion/attention/backends/flash_attn.py +++ b/vllm_omni/diffusion/attention/backends/flash_attn.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from fa3_fwd_interface import flash_attn_func, flash_attn_varlen_func from vllm.logger import init_logger from vllm_omni.diffusion.attention.backends.abstract import ( @@ -10,7 +9,13 @@ from vllm_omni.diffusion.attention.backends.abstract import ( AttentionImpl, AttentionMetadata, ) -from vllm_omni.diffusion.attention.backends.utils.fa import _pad_input, _unpad_input, _upad_input +from vllm_omni.diffusion.attention.backends.utils.fa import ( + _pad_input, + _unpad_input, + _upad_input, + flash_attn_func, + flash_attn_varlen_func, +) logger = init_logger(__name__) diff --git a/vllm_omni/diffusion/attention/backends/utils/fa.py b/vllm_omni/diffusion/attention/backends/utils/fa.py index d89082e71..6a02435a7 100644 --- a/vllm_omni/diffusion/attention/backends/utils/fa.py +++ b/vllm_omni/diffusion/attention/backends/utils/fa.py @@ -15,6 +15,20 @@ import torch import torch.nn.functional as F +from vllm_omni.utils.platform_utils import is_rocm + +if is_rocm(): + from vllm._aiter_ops import is_aiter_found_and_supported + + # Choose to enable this by default on ROCm + # Whenever possible as it is the fastest backend + if is_aiter_found_and_supported(): + from aiter import flash_attn_func, flash_attn_varlen_func # noqa: F401 + else: + raise ImportError("Aiter is not found and supported on currentROCm device.") +else: + from fa3_fwd_interface import flash_attn_func, flash_attn_varlen_func # noqa: F401 + def _index_first_axis(tensor, indices): """ diff --git a/vllm_omni/diffusion/attention/selector.py b/vllm_omni/diffusion/attention/selector.py index 706e79e7b..ed5946098 100644 --- a/vllm_omni/diffusion/attention/selector.py +++ b/vllm_omni/diffusion/attention/selector.py @@ -81,6 +81,28 @@ def get_attn_backend(head_size: int) -> type[AttentionBackend]: "Falling back to TORCH_SDPA backend.""" ) backend_name = "TORCH_SDPA" + elif detect_device_type() == "cuda" and is_rocm(): + from vllm._aiter_ops import is_aiter_found_and_supported + + compute_capability = torch.cuda.get_device_capability() + major, minor = compute_capability + + # Choose to enable this by default on ROCm + # Whenever possible as it is the fastest backend + # is_aiter_found_and_supported() checks if aiter library is found + # and is aiter supported on the current platform + # aiter currently only is supported on gfx942 and gfx950 + # https://github.com/vllm-project/vllm/blob/main/vllm/_aiter_ops.py + if is_aiter_found_and_supported() and 90 < major * 10 + minor < 100: + if backend_name is None: + backend_name = "FLASH_ATTN" + else: + if backend_name == "FLASH_ATTN": + logger.warning( + "Flash Attention requires `aiter` library which is only supported " + "on gfx942 and gfx950. Falling back to TORCH_SDPA backend." + ) + backend_name = "TORCH_SDPA" if backend_name is not None: backend_name_upper = backend_name.upper() From 2cc4375df840281b3326c2bd5f71765698eb7aba Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Tue, 27 Jan 2026 20:15:56 +0800 Subject: [PATCH 28/83] [Test] Add chunk e2e test case for CI (#956) Signed-off-by: wangyu31577 Co-authored-by: wangyu31577 --- .buildkite/pipeline.yml | 5 +- tests/conftest.py | 390 +++++++++++++++--- tests/e2e/offline_inference/conftest.py | 4 +- .../e2e/offline_inference/test_qwen3_omni.py | 2 + tests/e2e/online_serving/test_qwen3_omni.py | 74 +++- .../test_qwen3_omni_expansion.py | 26 +- tests/e2e/stage_configs/qwen3_omni_ci.yaml | 2 +- 7 files changed, 414 insertions(+), 89 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8f942b041..dcc9b3064 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -170,7 +170,10 @@ steps: depends_on: image-build commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_async_omni.py + - export VLLM_TEST_CLEAN_GPU_MEMORY="1" + - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py + - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py + - pytest -s -v tests/e2e/online_serving/test_async_omni.py agents: queue: "mithril-h100-pool" plugins: diff --git a/tests/conftest.py b/tests/conftest.py index 09696658d..ace891db7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,7 @@ import psutil import pytest import torch import yaml +from vllm.distributed.parallel_state import cleanup_dist_env_and_memory from vllm.logger import init_logger from vllm.utils.network_utils import get_open_port @@ -52,35 +53,118 @@ def default_vllm_config(): @pytest.fixture(autouse=True) def clean_gpu_memory_between_tests(): - clean_gpu_memory() + print("\n=== PRE-TEST GPU CLEANUP ===") + _run_pre_test_cleanup() + yield + _run_post_test_cleanup() -def clean_gpu_memory(): +def _run_pre_test_cleanup(): if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1": - yield + print("GPU cleanup disabled") return - # Wait for GPU memory to be cleared before starting the test - import gc - - from tests.utils import wait_for_gpu_memory_to_clear + print("Pre-test GPU status:") + _print_simple_gpu_status() num_gpus = torch.cuda.device_count() if num_gpus > 0: try: + from tests.utils import wait_for_gpu_memory_to_clear + wait_for_gpu_memory_to_clear( devices=list(range(num_gpus)), threshold_ratio=0.1, ) - except ValueError as e: - logger.info("Failed to clean GPU memory: %s", e) + except Exception as e: + print(f"Pre-test cleanup note: {e}") - yield - # Clean up GPU memory after the test +def _run_post_test_cleanup(): + if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1": + return + + import gc + if torch.cuda.is_available(): - torch.cuda.empty_cache() gc.collect() + torch.cuda.empty_cache() + + print("Post-test GPU status:") + _print_simple_gpu_status() + _print_gpu_processes() + + +def _print_simple_gpu_status(): + """Print simple GPU memory status""" + if not torch.cuda.is_available(): + print(" CUDA not available") + return + + num_devices = torch.cuda.device_count() + for device_id in range(num_devices): + try: + torch.cuda.set_device(device_id) + allocated = torch.cuda.memory_allocated(device_id) / (1024**2) + reserved = torch.cuda.memory_reserved(device_id) / (1024**2) + print(f" GPU {device_id}: Allocated: {allocated:.1f}MB, Reserved: {reserved:.1f}MB") + except Exception: + print(f" GPU {device_id}: Error reading status") + + +def _print_gpu_processes(): + """Print GPU information including nvidia-smi and system processes""" + + print("\n" + "=" * 80) + print("NVIDIA GPU Information (nvidia-smi)") + print("=" * 80) + + try: + nvidia_result = subprocess.run( + ["nvidia-smi"], + capture_output=True, + text=True, + timeout=5, + ) + + if nvidia_result.returncode == 0: + lines = nvidia_result.stdout.strip().split("\n") + for line in lines[:20]: + print(line) + + if len(lines) > 20: + print(f"... (showing first 20 of {len(lines)} lines)") + else: + print("nvidia-smi command failed") + + except (subprocess.TimeoutExpired, FileNotFoundError): + print("nvidia-smi not available or timed out") + except Exception as e: + print(f"Error running nvidia-smi: {e}") + + print("\n" + "=" * 80) + print("Detailed GPU Processes (nvidia-smi pmon)") + print("=" * 80) + + try: + pmon_result = subprocess.run( + ["nvidia-smi", "pmon", "-c", "1"], + capture_output=True, + text=True, + timeout=3, + ) + + if pmon_result.returncode == 0 and pmon_result.stdout.strip(): + print(pmon_result.stdout) + else: + print("No active GPU processes found via nvidia-smi pmon") + + except Exception: + print("nvidia-smi pmon not available") + + print("\n" + "=" * 80) + print("System Processes with GPU keywords") + print("=" * 80) def dummy_messages_from_mix_data( @@ -541,75 +625,238 @@ def convert_audio_to_text(audio_data): def modify_stage_config( yaml_path: str, - stage_updates: dict[int, dict[str, Any]], + updates: dict[str, Any], + deletes: dict[str, Any] = None, ) -> str: """ - Batch modify configurations for multiple stages in a YAML file. + Modify configurations in a YAML file, supporting both top-level and stage-specific modifications, + including addition, modification, and deletion of configurations. Args: yaml_path: Path to the YAML configuration file. - stage_updates: Dictionary where keys are stage IDs and values are dictionaries of - modifications for that stage. Each modification dictionary uses - dot-separated paths as keys and new configuration values as values. - Example: { - 0: {'engine_args.max_model_len': 5800}, - 1: {'runtime.max_batch_size': 2} - } + updates: Dictionary containing both top-level and stage-specific modifications to add or update. + Format: { + 'async_chunk': True, + 'stage_args': { + 0: {'engine_args.max_model_len': 5800}, + 1: {'runtime.max_batch_size': 2} + } + } + deletes: Dictionary containing configurations to delete. + Format: { + 'old_config': None, # Delete entire key + 'stage_args': { + 0: ['engine_args.old_param'], + 1: ['runtime.unused_setting'] + } + } Returns: str: Path to the newly created modified YAML file with timestamp suffix. - - Example: - >>> output_file = modify_stage_config( - ... 'config.yaml', - ... { - ... 0: {'engine_args.max_model_len': 5800}, - ... 1: {'runtime.max_batch_size': 2} - ... } - ... ) - >>> print(f"Modified configuration saved to: {output_file}") - Modified configuration saved to: config_1698765432.yaml """ path = Path(yaml_path) if not path.exists(): raise FileNotFoundError(f"yaml does not exist: {path}") + try: with open(yaml_path, encoding="utf-8") as f: config = yaml.safe_load(f) or {} except Exception as e: raise ValueError(f"Cannot parse YAML file: {e}") - stage_args = config.get("stage_args", []) - if not stage_args: - raise ValueError("the stage_args does not exist") + # Helper function to apply update + def apply_update(config_dict: dict, key_path: str, value: Any) -> None: + """Apply update to dictionary using dot-separated path.""" + # Handle direct list assignment (e.g., engine_input_source: [1, 2]) + if "." not in key_path: + # Simple key, set directly + config_dict[key_path] = value + return - for stage_id, config_dict in stage_updates.items(): - target_stage = None - for stage in stage_args: - if stage.get("stage_id") == stage_id: - target_stage = stage - break + current = config_dict + keys = key_path.split(".") - if target_stage is None: - available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s] - raise KeyError(f"Stage ID {stage_id} is not exist, available IDs: {available_ids}") + for i in range(len(keys) - 1): + key = keys[i] - for key_path, value in config_dict.items(): - current = target_stage - keys = key_path.split(".") - for i in range(len(keys) - 1): - key = keys[i] + # Handle list indices + if key.isdigit() and isinstance(current, list): + index = int(key) + if index < 0: + raise ValueError(f"Negative list index not allowed: {index}") + if index >= len(current): + # Expand list if needed + while len(current) <= index: + # If we need to go deeper (more keys after this), create a dict + # Otherwise, create None placeholder + current.append({} if i < len(keys) - 2 else None) + current = current[index] + elif isinstance(current, dict): + # Handle dictionary keys if key not in current: - raise KeyError(f"the {'.'.join(keys[: i + 1])} does not exist") - - elif not isinstance(current[key], dict) and i < len(keys) - 2: - raise ValueError(f"{'.'.join(keys[: i + 1])}' cannot continue deeper because it's not a dict") + # If there are more keys after this, create appropriate structure + if i < len(keys) - 1: + # Check if next key is a digit (list index) or string (dict key) + if keys[i + 1].isdigit(): + current[key] = [] + else: + current[key] = {} + else: + # This is the last key, create based on value type + current[key] = [] if isinstance(value, list) else {} + elif not isinstance(current[key], (dict, list)) and i < len(keys) - 1: + # If current value is not dict/list but we need to go deeper, replace it + if keys[i + 1].isdigit(): + current[key] = [] + else: + current[key] = {} current = current[key] - current[keys[-1]] = value + else: + # Current is not a dict or list, cannot traverse further + raise TypeError( + f"Cannot access {'.'.join(keys[: i + 1])} as a dict/list. It's a {type(current).__name__}" + ) + + # Set the final value + last_key = keys[-1] + if isinstance(current, list) and last_key.isdigit(): + # Setting a value in a list by index + index = int(last_key) + if index < 0: + raise ValueError(f"Negative list index not allowed: {index}") + if index >= len(current): + # Expand list if needed + while len(current) <= index: + current.append(None) + current[index] = value + elif isinstance(current, dict): + # Special case: if the value is a list and we're setting a top-level key + # Example: updating engine_input_source with [1, 2] + current[last_key] = value + else: + # Current is not a dict, cannot set key + raise TypeError(f"Cannot set value at {key_path}. Current type is {type(current).__name__}, expected dict.") + + # Helper function to delete by path + def delete_by_path(config_dict: dict, path: str) -> None: + """Delete configuration by dot-separated path.""" + if not path: + return + + current = config_dict + keys = path.split(".") + + # Traverse to the parent + for i in range(len(keys) - 1): + key = keys[i] + + # Handle list indices + if key.isdigit() and isinstance(current, list): + index = int(key) + if index < 0 or index >= len(current): + raise KeyError(f"List index {index} out of bounds") + current = current[index] + elif isinstance(current, dict): + if key not in current: + raise KeyError(f"Path {'.'.join(keys[: i + 1])} does not exist") + current = current[key] + else: + raise TypeError( + f"Cannot access {'.'.join(keys[: i + 1])} as a dict/list. It's a {type(current).__name__}" + ) + + # Delete the item + last_key = keys[-1] + + if isinstance(current, list) and last_key.isdigit(): + index = int(last_key) + if index < 0 or index >= len(current): + raise KeyError(f"List index {index} out of bounds") + del current[index] + elif isinstance(current, dict) and last_key in current: + del current[last_key] + else: + raise KeyError(f"Path {path} does not exist") + + # Apply deletions first + if deletes: + for key, value in deletes.items(): + if key == "stage_args": + if value and isinstance(value, dict): + stage_args = config.get("stage_args", []) + if not stage_args: + raise ValueError("stage_args does not exist in config") + + for stage_id, delete_paths in value.items(): + if not delete_paths: + continue + + # Find stage by ID + target_stage = None + for stage in stage_args: + if stage.get("stage_id") == stage_id: + target_stage = stage + break + + if target_stage is None: + available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s] + raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}") + + # Delete specified paths in this stage + for path in delete_paths: + if path: # Skip empty paths + delete_by_path(target_stage, path) + elif "." in key: + # Delete using dot-separated path + delete_by_path(config, key) + elif value is None and key in config: + # Delete entire key + del config[key] + + # Apply updates + for key, value in updates.items(): + if key == "stage_args": + if value and isinstance(value, dict): + stage_args = config.get("stage_args", []) + if not stage_args: + raise ValueError("stage_args does not exist in config") + + for stage_id, stage_updates in value.items(): + # Find stage by ID + target_stage = None + for stage in stage_args: + if stage.get("stage_id") == stage_id: + target_stage = stage + break + + if target_stage is None: + available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s] + raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}") + + # Apply updates to this stage + for path, val in stage_updates.items(): + # Check if this is a simple key (not dot-separated) + # Example: 'engine_input_source' vs 'engine_args.max_model_len' + if "." not in path: + # Direct key assignment (e.g., updating a list value) + target_stage[path] = val + else: + # Dot-separated path (e.g., nested dict access) + apply_update(target_stage, path, val) + elif "." in key: + # Apply using dot-separated path + apply_update(config, key, value) + else: + # Direct top-level key + config[key] = value + + # Save to new file with timestamp + timestamp = int(time.time()) + base_name = yaml_path.rsplit(".", 1)[0] if "." in yaml_path else yaml_path + output_path = f"{base_name}_{timestamp}.yaml" - output_path = f"{yaml_path.split('.')[0]}_{int(time.time())}.yaml" with open(output_path, "w", encoding="utf-8") as f: - yaml.dump(config, f, default_flow_style=False, sort_keys=False, allow_unicode=True, indent=2) + yaml.dump(config, f, default_flow_style=None, sort_keys=False, allow_unicode=True, indent=2) return output_path @@ -676,24 +923,32 @@ class OmniServer: raise RuntimeError(f"Server failed to start within {max_wait} seconds") def _kill_process_tree(self, pid): - """kill process and its children""" + """kill process and its children with verification""" try: parent = psutil.Process(pid) children = parent.children(recursive=True) + + # Get all PIDs first + all_pids = [pid] + [child.pid for child in children] + + # Terminate children for child in children: try: child.terminate() except psutil.NoSuchProcess: pass + # Wait for children gone, still_alive = psutil.wait_procs(children, timeout=10) + # Kill remaining children for child in still_alive: try: child.kill() except psutil.NoSuchProcess: pass + # Terminate parent try: parent.terminate() parent.wait(timeout=10) @@ -703,6 +958,24 @@ class OmniServer: except psutil.NoSuchProcess: pass + # VERIFICATION: Check if all processes are gone + time.sleep(1) # Give system time + alive_processes = [] + for check_pid in all_pids: + if psutil.pid_exists(check_pid): + alive_processes.append(check_pid) + + if alive_processes: + print(f"Warning: Processes still alive: {alive_processes}") + # Optional: Try system kill + import subprocess + + for alive_pid in alive_processes: + try: + subprocess.run(["kill", "-9", str(alive_pid)], timeout=2) + except Exception as e: + print(f"Cleanup failed: {e}") + except psutil.NoSuchProcess: pass @@ -713,4 +986,5 @@ class OmniServer: def __exit__(self, exc_type, exc_val, exc_tb): if self.proc: self._kill_process_tree(self.proc.pid) - clean_gpu_memory() + _run_post_test_cleanup() + cleanup_dist_env_and_memory() diff --git a/tests/e2e/offline_inference/conftest.py b/tests/e2e/offline_inference/conftest.py index 276fd8844..b3a4ade92 100644 --- a/tests/e2e/offline_inference/conftest.py +++ b/tests/e2e/offline_inference/conftest.py @@ -10,7 +10,7 @@ import pytest from vllm.distributed.parallel_state import cleanup_dist_env_and_memory from vllm.sampling_params import SamplingParams -from tests.conftest import clean_gpu_memory +from tests.conftest import _run_post_test_cleanup from vllm_omni.entrypoints.omni import Omni from vllm_omni.outputs import OmniRequestOutput @@ -336,7 +336,7 @@ class OmniRunner: self.close() del self.omni cleanup_dist_env_and_memory() - clean_gpu_memory() + _run_post_test_cleanup() def close(self): """Close and cleanup the Omni instance.""" diff --git a/tests/e2e/offline_inference/test_qwen3_omni.py b/tests/e2e/offline_inference/test_qwen3_omni.py index 14d117457..edd5958a0 100644 --- a/tests/e2e/offline_inference/test_qwen3_omni.py +++ b/tests/e2e/offline_inference/test_qwen3_omni.py @@ -7,6 +7,8 @@ E2E offline tests for Omni model with video input and audio output. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" + from pathlib import Path import pytest diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index c458a1686..8564104a0 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -7,8 +7,10 @@ E2E Online tests for Qwen3-Omni model with video input and audio output. import os os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" +os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0" import concurrent.futures +import threading import time from pathlib import Path @@ -23,36 +25,67 @@ from tests.conftest import ( generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video, + modify_stage_config, ) from vllm_omni.utils import is_rocm models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] + +def get_default_config(): + return str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml") + + +def get_chunk_config(): + path = modify_stage_config( + get_default_config(), + updates={ + "async_chunk": True, + "stage_args": { + 0: { + "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk" + }, + 1: { + "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk" + }, + }, + }, + deletes={"stage_args": {2: ["custom_process_input_func"]}}, + ) + return path + + +CHUNK_CONFIG_PATH = get_chunk_config() # CI stage config for 2xH100-80G GPUs or AMD GPU MI325 if is_rocm(): # ROCm stage config optimized for MI325 GPU stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")] else: - stage_configs = [str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")] + stage_configs = [get_default_config(), CHUNK_CONFIG_PATH] # Create parameter combinations for model and stage config test_params = [(model, stage_config) for model in models for stage_config in stage_configs] +_omni_server_lock = threading.Lock() + + @pytest.fixture(scope="module") def omni_server(request): """Start vLLM-Omni server as a subprocess with actual model weights. Uses session scope so the server starts only once for the entire test session. Multi-stage initialization can take 10-20+ minutes. """ - model, stage_config_path = request.param + with _omni_server_lock: + model, stage_config_path = request.param - print(f"Starting OmniServer with model: {model}") - print("This may take 10-20+ minutes for initialization...") + print(f"Starting OmniServer with model: {model}") + + with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server: + print("OmniServer started successfully") + yield server + print("OmniServer stopping...") - with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server: - print("OmniServer started successfully") - yield server print("OmniServer stopped") @@ -113,7 +146,7 @@ def get_max_batch_size(size_type="few"): @pytest.mark.skipif(is_rocm(), reason="Test skipped on AMD environment due to known output issues") @pytest.mark.parametrize("omni_server", test_params, indirect=True) -def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server) -> None: +def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None: """ Test multi-modal input processing and text/audio output generation via OpenAI API. Deploy Setting: default yaml @@ -122,6 +155,11 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server) -> None: Input Setting: stream=True Datasets: single request """ + # TODO：This skip will be removed when the chunk scenario supports multimodal input. + param = request.node.callspec.params.get("omni_server") + + if param[1] == CHUNK_CONFIG_PATH: + pytest.skip("The current chunk scenario does not support multimodal.") # Test single completion e2e_list = list() @@ -223,16 +261,18 @@ def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None: assert len(chat_completions) == num_concurrent_requests, "Not all requests succeeded." for chat_completion in chat_completions: # Verify audio output success - audio_message = chat_completion.choices[1].message - audio_data = audio_message.audio.data - assert audio_data is not None, "No audio output is generated" - assert audio_message.audio.expires_at > time.time(), "The generated audio has expired." + audio_data = None + text_content = None + for choice in chat_completion.choices: + if choice.message.audio is not None: + audio_message = choice.message + audio_data = audio_message.audio.data + assert audio_message.audio.expires_at > time.time(), "The generated audio has expired." - # Verify text output success - text_choice = chat_completion.choices[0] - text_content = text_choice.message.content - assert text_choice.message.content is not None, "No text output is generated" - assert "beijing" in text_choice.message.content.lower(), "The output do not contain keywords." + if choice.message.content is not None: + # Verify text output success + text_content = choice.message.content + assert "beijing" in text_content.lower(), "The output do not contain keywords." # Verify text output same as audio output audio_content = convert_audio_to_text(audio_data) diff --git a/tests/e2e/online_serving/test_qwen3_omni_expansion.py b/tests/e2e/online_serving/test_qwen3_omni_expansion.py index 563eaf628..5db07fd89 100644 --- a/tests/e2e/online_serving/test_qwen3_omni_expansion.py +++ b/tests/e2e/online_serving/test_qwen3_omni_expansion.py @@ -73,12 +73,14 @@ def get_max_batch_size(size_type="few"): def get_deploy_config(deploy_type="TP1"): result = { "TP1": { - 0: { - "engine_args.gpu_memory_utilization": 0.95, - "engine_args.tensor_parallel_size": 1, - "runtime.devices": "0", - }, - 2: {"runtime.devices": "1"}, + "stage_args": { + 0: { + "engine_args.gpu_memory_utilization": 0.95, + "engine_args.tensor_parallel_size": 1, + "runtime.devices": "0", + }, + 2: {"runtime.devices": "1"}, + } } } return result.get(deploy_type, result["TP1"]) @@ -151,8 +153,10 @@ def test_audio_to_text_audio_001(test_config: tuple[str, str]) -> None: stage_config_path = modify_stage_config( stage_config_path, { - 0: {"runtime.max_batch_size": num_concurrent_requests}, - 1: {"runtime.max_batch_size": num_concurrent_requests}, + "stage_args": { + 0: {"runtime.max_batch_size": num_concurrent_requests}, + 1: {"runtime.max_batch_size": num_concurrent_requests}, + } }, ) with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server: @@ -246,8 +250,10 @@ def test_image_to_text_audio_001(test_config: tuple[str, str]) -> None: stage_config_path = modify_stage_config( stage_config_path, { - 0: {"runtime.max_batch_size": num_concurrent_requests}, - 1: {"runtime.max_batch_size": num_concurrent_requests}, + "stage_args": { + 0: {"runtime.max_batch_size": num_concurrent_requests}, + 1: {"runtime.max_batch_size": num_concurrent_requests}, + } }, ) with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "90"]) as server: diff --git a/tests/e2e/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/stage_configs/qwen3_omni_ci.yaml index 4b5638529..60343dfdd 100644 --- a/tests/e2e/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/stage_configs/qwen3_omni_ci.yaml @@ -79,7 +79,7 @@ stage_args: engine_output_type: audio # Final output: audio waveform gpu_memory_utilization: 0.1 distributed_executor_backend: "mp" - max_num_batched_tokens: 1000000 + max_num_batched_tokens: 100000 hf_config_name: thinker_config async_scheduling: false engine_input_source: [1] From 16be03b909d2f10763837377a95fdb482db08f27 Mon Sep 17 00:00:00 2001 From: yangsonglin13 Date: Tue, 27 Jan 2026 20:17:02 +0800 Subject: [PATCH 29/83] [Feature] Implement YuanrongConnector based on OmniConnectorBase (#716) Signed-off-by: yangsonglin13 --- .../design/feature/disaggregated_inference.md | 214 +++++------------- .../omni_connectors/mooncake_connector.md | 74 ++++++ .../shared_memory_connector.md | 27 +++ .../omni_connectors/yuanrong_connector.md | 101 +++++++++ vllm_omni/distributed/__init__.py | 2 + .../distributed/omni_connectors/__init__.py | 2 + .../connectors/mooncake_connector.py | 12 +- .../connectors/yuanrong_connector.py | 132 +++++++++++ .../distributed/omni_connectors/factory.py | 13 ++ .../omni_connectors/utils/config.py | 2 +- .../qwen2_5_omni_multiconnector.yaml | 9 +- 11 files changed, 425 insertions(+), 163 deletions(-) create mode 100644 docs/design/feature/omni_connectors/mooncake_connector.md create mode 100644 docs/design/feature/omni_connectors/shared_memory_connector.md create mode 100644 docs/design/feature/omni_connectors/yuanrong_connector.md create mode 100644 vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py diff --git a/docs/design/feature/disaggregated_inference.md b/docs/design/feature/disaggregated_inference.md index 2dae03ff8..57cd39d16 100644 --- a/docs/design/feature/disaggregated_inference.md +++ b/docs/design/feature/disaggregated_inference.md @@ -1,27 +1,35 @@ # Disaggregated Inference for Omni-Modality Models -This guide explains how to configure and use distributed connectors (vllm_omni/distributed/connectors) in vllm-omni for multi-stage pipelines. +This guide explains how to configure and use distributed connectors +(`vllm_omni/distributed/omni_connectors`) in vllm-omni for multi-stage pipelines. -## 1. Overview +Backend-specific setup lives in separate docs: + +- [SharedMemoryConnector](omni_connectors/shared_memory_connector.md) +- [MooncakeConnector](omni_connectors/mooncake_connector.md) +- [YuanrongConnector](omni_connectors/yuanrong_connector.md) + +## Overview Connectors enable data transfer between pipeline stages (e.g., Thinker -> Talker). -Currently supported connectors operate in **D2H2D (Device-to-Host-to-Device)** mode: -1. **SharedMemoryConnector**: Uses system shared memory. -2. **MooncakeConnector**: Uses [Mooncake](https://github.com/kvcache-ai/Mooncake). +Current connectors operate in D2H2D (device to host to device) mode. -* **SharedMemoryConnector (Default)**: Zero-copy (on host), lowest latency. Best for **single-node** deployments. Auto-configured if no connectors are specified. -* **MooncakeConnector**: TCP/RDMA based. Best for **multi-node** distributed deployments. Requires a Mooncake Master service. +## Connector Choices -## 2. API Design +| Use Case | Recommended Connector | Notes | +| :--- | :--- | :--- | +| Single node | SharedMemoryConnector | Auto-configured if no connector is specified. | +| Multi node (Mooncake) | MooncakeConnector | Requires Mooncake Master + metadata server. | +| Multi node (Yuanrong) | YuanrongConnector | Requires Yuanrong Datasystem + etcd. | -The connector system is built around the `OmniConnectorBase` abstraction, which decouples data transport from stage logic. +## Core API -### Core Interface +The connector system is built around `OmniConnectorBase`. ```python class OmniConnectorBase(ABC): @abstractmethod - def put(self, from_stage: str, to_stage: str, request_id: str, data: Any) -> tuple[bool, int, Optional[dict]]: + def put(self, from_stage: str, to_stage: str, put_key: str, data: Any) -> tuple[bool, int, Optional[dict]]: """ Store data. Returns: (success, serialized_size, metadata) @@ -29,145 +37,23 @@ class OmniConnectorBase(ABC): pass @abstractmethod - def get(self, from_stage: str, to_stage: str, request_id: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]: + def get(self, from_stage: str, to_stage: str, get_key: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]: """ Retrieve data. - Args: metadata - Transport-specific handles returned by put() (e.g., SHM name). + Args: metadata - transport-specific handles returned by put() (e.g., SHM name). Returns: (object, serialized_size) """ pass ``` -### Key Concept: Metadata Passing -Unlike a pure key-value store, some connectors (like `SharedMemoryConnector`) generate transient resources (e.g., a shared memory block name) during `put()`. This `metadata` **must be passed via the control plane** (e.g., HTTP headers, queue messages) from the producer stage to the consumer stage so `get()` can locate the data. +### Metadata Passing -## 3. Backends & Use Cases +Some connectors (e.g., SharedMemoryConnector) generate transient resources during `put()`. +This `metadata` must be passed through the control plane so `get()` can locate the data. -### 3.1 SharedMemoryConnector -**Best for:** Single-node, high-performance IPC. +## Configuration Model -* **Mechanism:** - * **Small Payloads (< Threshold)**: Data is serialized and passed directly "inline" within the `metadata` dictionary. This avoids the overhead of creating SHM blocks for tiny messages. - * **Large Payloads (>= Threshold)**: Data is written to a named System Shared Memory block. The block name is returned in `metadata`. -* **Configuration:** - * `shm_threshold_bytes`: Size in bytes to switch from inline to SHM (default: 64KB). - -### 3.2 MooncakeConnector -**Best for:** Multi-node distributed inference. - -* **Mechanism:** Uses Mooncake's distributed KVCache store. - * **Data Plane**: TCP or RDMA for high-bandwidth transfer. - * **Control Plane**: Uses a centralized Mooncake Master and Metadata Server. - * **Keying**: Deterministic keys based on `request_id/from_stage_to_stage`. -* **Requirements**: Requires a running Mooncake Master service. - -## 4. Relationship with vLLM - -vLLM provides specialized distributed mechanisms for specific artifacts: -* **KV Transfer** (`vllm.distributed.kv_transfer`): Optimized for transferring KV caches between prefill and decode instances (using NCCL, Mooncake, etc.). -* **EC Transfer** (`vllm.distributed.ec_transfer`): Optimized for sharing encoder embeddings. -* **Device Communicators** (`vllm.distributed.device_communicators`): Low-level primitives (NCCL, SHM) for Tensor/Pipeline Parallelism. - -`vllm-omni` complements this by introducing a **Generalized Connector Abstraction** (`OmniConnector`) for multimodal pipelines. While vLLM's connectors are artifact-specific, `vllm-omni`: - -1. **Unifies Transport**: Provides a single API (`put`/`get`) to transport *any* stage artifact (Input Embeddings, Hidden States, Audio/Image Tensors, KV Cache, Final Output) between arbitrary pipeline stages (e.g., AudioEncoder -> LLM -> AudioGenerator). -2. **Extends Connectivity**: Enables flexible construction of complex DAGs (Directed Acyclic Graphs) where stages can run in the same process, same node, or across nodes, using the most appropriate backend (SHM, Mooncake, etc.) for each edge. -3. **Wraps & Adapts**: Can internally utilize vLLM's specialized `kv_transfer` for KV paths while using generic transports (SHM/Mooncake) for other data types, presenting a consistent interface to the application layer. - -## 5. Installation (Mooncake) - -If using `MooncakeConnector`, install the library first: - -```bash -# For CUDA-enabled systems (Recommended) -pip install mooncake-transfer-engine - -# For non-CUDA systems -pip install mooncake-transfer-engine-non-cuda -``` - -## 6. Using MooncakeConnector - -### 6.1 Start Mooncake Master - -Start the master service on your primary node: - -```bash -# if you use mooncake SSD storage -mkdir -p ./mc_storage - -mooncake_master \ - --rpc_port=50051 \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port=8080 \ - --metrics_port=9003 \ - --root_fs_dir=./mc_storage/ \ - --cluster_id=mc-local-1 & -``` - -### 6.2 Configuration (YAML) - -Edit your stage config (e.g., `qwen2_5_omni.yaml`). - -**Step 1: Define Connector in Global Runtime** - -```yaml -runtime: - connectors: - connector_of_mooncake: - name: MooncakeConnector - extra: - host: "127.0.0.1" # Local Worker IP - metadata_server: "http://:8080/metadata" - master: ":50051" - segment: 512000000 # 512MB segment - localbuf: 64000000 # 64MB buffer - proto: "tcp" # "tcp" or "rdma" - ``` - -**Mooncake Configuration Parameters:** - -* **host**: The hostname or IP address of the local machine (worker). Mooncake uses this to register itself in the metadata server so other nodes can find it. -* **metadata_server**: The URL of the metadata server. This is used for service discovery and connection establishment (e.g., exchanging QP information for RDMA). -* **master**: The address of the Mooncake Master Server (e.g., `:50051`). This is used for global state management and control plane operations. -* **segment**: The size of the global memory segment in bytes (default: ~512MB). This defines the shared memory region accessible by Mooncake for data transfer. -* **localbuf**: The size of the local buffer in bytes (default: ~64MB). Used for local data buffering during transfer operations. -* **proto**: The transport protocol to use. Options: - * `tcp`: Standard TCP/IP (easier setup, universal compatibility). - * `rdma`: Remote Direct Memory Access (higher performance, requires RDMA-capable hardware). - -For more details, refer to the [Mooncake Repository](https://github.com/kvcache-ai/Mooncake). - - **Step 2: Reference in Stages** - -Explicitly link stages using `input_connectors` and `output_connectors`: - -```yaml -stage_args: - - stage_id: 0 - # ... - output_connectors: - to_stage_1: connector_of_mooncake - - - stage_id: 1 - # ... - input_connectors: - from_stage_0: connector_of_mooncake -``` - -## 7. Using SharedMemoryConnector (Auto-Mode) - -**Best for single-node.** - -The system will automatically create `SharedMemoryConnector`s for any pipeline edge that does not have an explicit connector defined. This is inferred from: -1. `runtime.edges` list in the config. -2. `engine_input_source` dependencies defined in `stage_args`. - -### Threshold Configuration -By default, payloads larger than **64KB** (default threshold) are transferred via shared memory, while smaller ones use the control queue (inline). - -To adjust this threshold (e.g., to 1GB), add the following to your `runtime.connectors`: +Define connectors in runtime: ```yaml runtime: @@ -175,26 +61,46 @@ runtime: connector_of_shared_memory: name: SharedMemoryConnector extra: - shm_threshold_bytes: 1024 # 1KB threshold + shm_threshold_bytes: 65536 ``` -## 8. Summary +Wire stages to connectors: -| Use Case | Recommended Connector | Configuration | -| :--- | :--- | :--- | -| **Single Node** | `SharedMemoryConnector` | **None** (Automatic) or Custom Threshold | -| **Multi Node** | `MooncakeConnector` | Explicit YAML + Mooncake Master | +```yaml +stage_args: + - stage_id: 0 + output_connectors: + to_stage_1: connector_of_shared_memory -## 9. Operational Notes (important) + - stage_id: 1 + input_connectors: + from_stage_0: connector_of_shared_memory +``` -- **Fail-fast config validation**: the loader raises if any expected edge is missing a connector. Define `input_connectors`/`output_connectors` or rely on auto-SHM filling; otherwise startup aborts. -- **Missing payloads halt the stage**: workers expect connector payloads; if metadata or connector config is missing, the stage raises and stops. Verify connector wiring and metadata propagation before production. +If a pipeline edge has no explicit connector, the system auto-creates a +SharedMemoryConnector for that edge. -## 10. Future Roadmap: Device-to-Device (D2D) Transport +## Relationship with vLLM -The current implementations (`SharedMemoryConnector`, `MooncakeConnector`) utilize a **D2H2D (Device-to-Host-to-Device)** data path. Tensors are moved to CPU memory (Host) for transport, which incurs PCIe overhead. +vLLM provides specialized distributed mechanisms for specific artifacts: -As outlined in the design RFC, future versions will introduce **D2D (Device-to-Device)** connectors: +- KV Transfer (`vllm.distributed.kv_transfer`): optimized for KV caches. +- EC Transfer (`vllm.distributed.ec_transfer`): optimized for encoder embeddings. +- Device Communicators (`vllm.distributed.device_communicators`): low-level primitives (NCCL, SHM). -* **Goal**: Direct GPU-to-GPU transfer (via NCCL, UCX, or IPC) to minimize latency for large tensor payloads. -* **Mechanism**: The `OmniConnector` API allows `put()` to initiate a transfer and return a lightweight handle (metadata) via the control plane, while the heavy payload flows directly between devices. +vllm-omni complements this with a generalized connector abstraction: + +1. Unifies transport via a single `put`/`get` API for any stage artifact. +2. Enables DAG-style pipelines across processes or nodes with per-edge transports. +3. Can wrap vLLM-specific transfers for KV paths while keeping a consistent interface. + +## Operational Notes + +- Fail-fast config validation: missing expected edges cause startup failures. +- Missing payloads halt stages: verify connector wiring and metadata propagation. + +## Future Roadmap: D2D Transport + +Current connectors use D2H2D paths. Future versions will introduce direct +device-to-device connectors (NCCL, UCX, IPC) to reduce latency for large +tensor payloads. diff --git a/docs/design/feature/omni_connectors/mooncake_connector.md b/docs/design/feature/omni_connectors/mooncake_connector.md new file mode 100644 index 000000000..fee409ecf --- /dev/null +++ b/docs/design/feature/omni_connectors/mooncake_connector.md @@ -0,0 +1,74 @@ +# MooncakeConnector + +## When to Use + +Best for multi-node distributed inference using Mooncake. + +## Installation + +```bash +# For CUDA-enabled systems (recommended) +pip install mooncake-transfer-engine + +# For non-CUDA systems +pip install mooncake-transfer-engine-non-cuda +``` + +## Start Mooncake Master + +```bash +# If you use Mooncake SSD storage +mkdir -p ./mc_storage + +mooncake_master \ + --rpc_port=50051 \ + --enable_http_metadata_server=true \ + --http_metadata_server_host=0.0.0.0 \ + --http_metadata_server_port=8080 \ + --metrics_port=9003 \ + --root_fs_dir=./mc_storage/ \ + --cluster_id=mc-local-1 & +``` + +## Configuration + +Define the connector in runtime: + +```yaml +runtime: + connectors: + connector_of_mooncake: + name: MooncakeConnector + extra: + host: "127.0.0.1" + metadata_server: "http://:8080/metadata" + master: ":50051" + segment: 512000000 + localbuf: 64000000 + proto: "tcp" +``` + +Wire stages to the connector: + +```yaml +stage_args: + - stage_id: 0 + output_connectors: + to_stage_1: connector_of_mooncake + + - stage_id: 1 + input_connectors: + from_stage_0: connector_of_mooncake +``` + +Parameters: + +- host: local worker IP registered in the metadata server. +- metadata_server: metadata server URL for discovery and setup. +- master: Mooncake Master address. +- segment: global memory segment size in bytes. +- localbuf: local buffer size in bytes. +- proto: transport protocol ("tcp" or "rdma"). + +For more details, refer to the +[Mooncake repository](https://github.com/kvcache-ai/Mooncake). diff --git a/docs/design/feature/omni_connectors/shared_memory_connector.md b/docs/design/feature/omni_connectors/shared_memory_connector.md new file mode 100644 index 000000000..eb91a889a --- /dev/null +++ b/docs/design/feature/omni_connectors/shared_memory_connector.md @@ -0,0 +1,27 @@ +# SharedMemoryConnector + +## When to Use + +Best for single-node deployments where stages run on the same host. It is +auto-configured when no explicit connector is specified for an edge. + +## How It Works + +- Small payloads (< threshold): serialized and passed inline in metadata. +- Large payloads (>= threshold): stored in shared memory; the SHM name is + returned in metadata. + +## Configuration + +```yaml +runtime: + connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + shm_threshold_bytes: 65536 +``` + +## Notes + +- Auto-mode uses SharedMemoryConnector if no connector is declared for an edge. diff --git a/docs/design/feature/omni_connectors/yuanrong_connector.md b/docs/design/feature/omni_connectors/yuanrong_connector.md new file mode 100644 index 000000000..88989d31a --- /dev/null +++ b/docs/design/feature/omni_connectors/yuanrong_connector.md @@ -0,0 +1,101 @@ +# YuanrongConnector + +## When to Use + +Best for multi-node distributed inference using Yuanrong Datasystem. + +## Mechanism + +Uses Yuanrong Datasystem's distributed KV store (`datasystem.kv_client`). + +- Data Plane: TCP or RDMA for high-bandwidth transfer. +- Control Plane: Yuanrong Datasystem workers and etcd. +- Keying: deterministic keys based on `put_key` (often composed as `request_id:fromStage_toStage`). + +## Installation + +```bash +pip install openyuanrong-datasystem +``` + +## Start etcd + +```bash +# Download and install etcd (v3.5.12 or higher) +ETCD_VERSION="v3.5.12" +ETCD_ARCH="linux-arm64" +wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-${ETCD_ARCH}.tar.gz +tar -xvf etcd-${ETCD_VERSION}-${ETCD_ARCH}.tar.gz +cd etcd-${ETCD_VERSION}-${ETCD_ARCH} +sudo cp etcd etcdctl /usr/local/bin/ + +# Start etcd +etcd \ + --name etcd-single \ + --data-dir /tmp/etcd-data \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://0.0.0.0:2379 \ + --listen-peer-urls http://0.0.0.0:2380 \ + --initial-advertise-peer-urls http://0.0.0.0:2380 \ + --initial-cluster etcd-single=http://0.0.0.0:2380 & + +# Verify etcd is running +etcdctl --endpoints "127.0.0.1:2379" put key "value" +etcdctl --endpoints "127.0.0.1:2379" get key +``` + +For production environments, refer to the +[official etcd clustering documentation](https://etcd.io/docs/current/op-guide/clustering/). + +## Start Datasystem Worker + +```bash +# Replace ${ETCD_IP} with etcd node IP, ${WORKER_IP} with local node IP +dscli start -w \ + --worker_address "${WORKER_IP}:31501" \ + --etcd_address "${ETCD_IP}:2379" \ + --shared_memory_size_mb 20480 +``` + +To stop the worker: + +```bash +dscli stop --worker_address "${WORKER_IP}:31501" +``` + +## Configuration + +Define the connector in runtime: + +```yaml +runtime: + connectors: + connector_of_yuanrong: + name: YuanrongConnector + extra: + host: "127.0.0.1" + port: 31501 + get_sub_timeout_ms: 1000 +``` + +Wire stages to the connector: + +```yaml +stage_args: + - stage_id: 0 + output_connectors: + to_stage_1: connector_of_yuanrong + + - stage_id: 1 + input_connectors: + from_stage_0: connector_of_yuanrong +``` + +Parameters: + +- host: datasystem worker host. +- port: datasystem worker port. +- get_sub_timeout_ms: get timeout in milliseconds (0 for no timeout). + +For more details, refer to the +[Yuanrong Datasystem repository](https://atomgit.com/openeuler/yuanrong-datasystem). diff --git a/vllm_omni/distributed/__init__.py b/vllm_omni/distributed/__init__.py index dd58b0f22..3a7634219 100644 --- a/vllm_omni/distributed/__init__.py +++ b/vllm_omni/distributed/__init__.py @@ -8,6 +8,7 @@ from .omni_connectors import ( OmniConnectorFactory, OmniTransferConfig, SharedMemoryConnector, + YuanrongConnector, load_omni_transfer_config, ) @@ -20,6 +21,7 @@ __all__ = [ "OmniConnectorFactory", "MooncakeConnector", "SharedMemoryConnector", + "YuanrongConnector", # Utilities "load_omni_transfer_config", ] diff --git a/vllm_omni/distributed/omni_connectors/__init__.py b/vllm_omni/distributed/omni_connectors/__init__.py index b5288798c..fcec32eaa 100644 --- a/vllm_omni/distributed/omni_connectors/__init__.py +++ b/vllm_omni/distributed/omni_connectors/__init__.py @@ -4,6 +4,7 @@ from .connectors.base import OmniConnectorBase from .connectors.mooncake_connector import MooncakeConnector from .connectors.shm_connector import SharedMemoryConnector +from .connectors.yuanrong_connector import YuanrongConnector from .factory import OmniConnectorFactory from .utils.config import ConnectorSpec, OmniTransferConfig from .utils.initialization import ( @@ -26,6 +27,7 @@ __all__ = [ # Specific implementations "MooncakeConnector", "SharedMemoryConnector", + "YuanrongConnector", # Utilities "load_omni_transfer_config", "initialize_connectors_from_config", diff --git a/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py b/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py index 2e12d3a40..2f969b9bf 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py @@ -51,7 +51,7 @@ class MooncakeConnector(OmniConnectorBase): def _make_key(self, rid: str, from_stage: str, to_stage: str) -> str: """Generate store key for request between stages.""" - return f"{rid}/{from_stage}_to_{to_stage}" + return f"{rid}/{from_stage}_{to_stage}" def _init_store(self): """Initialize Mooncake store.""" @@ -72,16 +72,14 @@ class MooncakeConnector(OmniConnectorBase): # Use base class serialization methods for consistency - def put( - self, from_stage: str, to_stage: str, request_id: str, data: Any - ) -> tuple[bool, int, dict[str, Any] | None]: + def put(self, from_stage: str, to_stage: str, put_key: str, data: Any) -> tuple[bool, int, dict[str, Any] | None]: if not self.store: logger.error("Store not initialized") return False, 0, None try: serialized_data = self.serialize_obj(data) - key = self._make_key(request_id, from_stage, to_stage) + key = self._make_key(put_key, from_stage, to_stage) self.store.put(key, serialized_data, self.pin) self._metrics["puts"] += 1 @@ -102,7 +100,7 @@ class MooncakeConnector(OmniConnectorBase): return False, 0, None def get( - self, from_stage: str, to_stage: str, request_id: str, metadata: dict[str, Any] | None = None + self, from_stage: str, to_stage: str, get_key: str, metadata: dict[str, Any] | None = None ) -> tuple[Any, int] | None: if not self.store: logger.error("Store not initialized") @@ -110,7 +108,7 @@ class MooncakeConnector(OmniConnectorBase): retries = 20 sleep_s = 0.05 - key = self._make_key(request_id, from_stage, to_stage) + key = self._make_key(get_key, from_stage, to_stage) for attempt in range(retries): try: diff --git a/vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py b/vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py new file mode 100644 index 000000000..c147d00fd --- /dev/null +++ b/vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +from ..utils.logging import get_connector_logger +from .base import OmniConnectorBase + +logger = get_connector_logger(__name__) + +try: + from datasystem.kv_client import KVClient, SetParam, WriteMode +except ImportError: + logger.warning("Datasystem not available, YuanrongConnector will not work") + KVClient = None + + +class YuanrongConnector(OmniConnectorBase): + """Datasystem-based distributed connector for OmniConnector.""" + + def __init__(self, config: dict[str, Any]): + if KVClient is None: + raise ImportError("Datasystem not available") + + self.config = config + self.client = None + self.set_param = SetParam() + self.set_param.write_mode = WriteMode.NONE_L2_CACHE_EVICT + self.get_sub_timeout_ms = max(0, int(self.config.get("get_sub_timeout_ms", 1000))) + + self._metrics = { + "puts": 0, + "gets": 0, + "bytes_transferred": 0, + "errors": 0, + "timeouts": 0, + } + + self._init_client() + + def _make_key(self, rid: str, from_stage: str, to_stage: str) -> str: + """Generate key for request between stages.""" + return f"{rid}:{from_stage}_{to_stage}" + + def _init_client(self): + """Initialize Datasystem client.""" + try: + self.host = self.config.get("host", "127.0.0.1") + self.port = int(self.config.get("port", "35001")) + self.client = KVClient(self.host, self.port) + self.client.init() + + logger.info("YuanrongConnector initialized successfully") + except Exception as e: + logger.error("Failed to initialize Datasystem client: %s", e) + raise + + def put(self, from_stage: str, to_stage: str, put_key: str, data: Any) -> tuple[bool, int, dict[str, Any] | None]: + if not self.client: + logger.error("Datasystem client not initialized") + return False, 0, None + + try: + serialized_data = self.serialize_obj(data) + key = self._make_key(put_key, from_stage, to_stage) + self.client.set(key, serialized_data, self.set_param.write_mode) + + self._metrics["puts"] += 1 + self._metrics["bytes_transferred"] += len(serialized_data) + + logger.debug( + "YuanrongConnector: stored %s (%s -> %s) %d bytes", + key, + from_stage, + to_stage, + len(serialized_data), + ) + return True, len(serialized_data), None + + except Exception as exc: + self._metrics["errors"] += 1 + logger.error("YuanrongConnector put failed: %s", exc) + return False, 0, None + + def get( + self, from_stage: str, to_stage: str, get_key: str, metadata: dict[str, Any] | None = None + ) -> tuple[Any, int] | None: + if not self.client: + logger.error("Datasystem client not initialized") + return None + + key = self._make_key(get_key, from_stage, to_stage) + try: + raw_list = self.client.get([key], False, self.get_sub_timeout_ms) + raw_data = raw_list[0] if raw_list else None + if raw_data is not None: + data = self.deserialize_obj(raw_data) + self._metrics["gets"] += 1 + payload_size = len(raw_data) + logger.debug( + "YuanrongConnector: retrieved %s (%s -> %s) %d bytes", + key, + from_stage, + to_stage, + payload_size, + ) + return data, payload_size + + except Exception as exc: + self._metrics["timeouts"] += 1 + logger.error("YuanrongConnector get failed: %s", exc) + return None + + def cleanup(self, request_id: str) -> None: + if not self.client: + return + + # Note: Datasystem doesn't have explicit delete, data will be garbage collected + logger.debug("YuanrongConnector: cleanup requested for %s (no-op)", request_id) + + def health(self) -> dict[str, Any]: + if not self.client: + return {"status": "unhealthy", "error": "Datasystem client not initialized"} + + return {"status": "healthy", "host": self.host, "port": self.port, **self._metrics} + + def close(self) -> None: + if not self.client: + return + + self.client = None + logger.info("YuanrongConnector closed") diff --git a/vllm_omni/distributed/omni_connectors/factory.py b/vllm_omni/distributed/omni_connectors/factory.py index ee640d92f..34415f7fd 100644 --- a/vllm_omni/distributed/omni_connectors/factory.py +++ b/vllm_omni/distributed/omni_connectors/factory.py @@ -82,6 +82,19 @@ def _create_shm_connector(config: dict[str, Any]) -> OmniConnectorBase: return SharedMemoryConnector(config) +def _create_yuanrong_connector(config: dict[str, Any]) -> OmniConnectorBase: + try: + from .connectors.yuanrong_connector import YuanrongConnector + except ImportError: + import os + import sys + + sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + from omni_connectors.connectors.yuanrong_connector import YuanrongConnector + return YuanrongConnector(config) + + # Register connectors OmniConnectorFactory.register_connector("MooncakeConnector", _create_mooncake_connector) OmniConnectorFactory.register_connector("SharedMemoryConnector", _create_shm_connector) +OmniConnectorFactory.register_connector("YuanrongConnector", _create_yuanrong_connector) diff --git a/vllm_omni/distributed/omni_connectors/utils/config.py b/vllm_omni/distributed/omni_connectors/utils/config.py index 9316c7e11..5556296d9 100644 --- a/vllm_omni/distributed/omni_connectors/utils/config.py +++ b/vllm_omni/distributed/omni_connectors/utils/config.py @@ -13,7 +13,7 @@ logger = get_connector_logger(__name__) class ConnectorSpec: """Specification for a connector instance.""" - name: str # e.g., "MooncakeConnector", "SharedMemoryConnector" + name: str # e.g., "MooncakeConnector", "SharedMemoryConnector", "YuanrongConnector" extra: dict[str, Any] = field(default_factory=dict) # backend-specific config diff --git a/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml b/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml index f2e4d03dd..aae194d37 100644 --- a/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml @@ -117,8 +117,15 @@ runtime: localbuf: 64000000 # 64MB proto: "tcp" + # Yuanrong connector for cross-node/intra-node communication + yuanrong_connector: + name: YuanrongConnector + extra: + host: "127.0.0.1" + port: "35000" + # SharedMemory connector for intra-node communication - #Alternative SHM connector with different threshold + # Alternative SHM connector with different threshold shared_memory_connector: name: SharedMemoryConnector extra: From 2f5e32477c1e11d2ca710fcdf69942d58f885500 Mon Sep 17 00:00:00 2001 From: Yuanheng Zhao <54058983+yuanheng-zhao@users.noreply.github.com> Date: Tue, 27 Jan 2026 20:36:54 +0800 Subject: [PATCH 30/83] [NFC] Remove redundant torch.no_grad in models and pipelines (#854) Signed-off-by: yuanheng --- vllm_omni/diffusion/models/bagel/bagel_transformer.py | 5 ----- .../diffusion/models/flux2_klein/pipeline_flux2_klein.py | 1 - vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py | 1 - .../diffusion/models/stable_audio/pipeline_stable_audio.py | 1 - vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py | 1 - vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py | 1 - vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py | 1 - vllm_omni/diffusion/models/z_image/pipeline_z_image.py | 1 - .../models/qwen2_5_omni/qwen2_5_omni_token2wav.py | 2 -- 9 files changed, 14 deletions(-) diff --git a/vllm_omni/diffusion/models/bagel/bagel_transformer.py b/vllm_omni/diffusion/models/bagel/bagel_transformer.py index 7f36c0b69..6ee81a4fd 100644 --- a/vllm_omni/diffusion/models/bagel/bagel_transformer.py +++ b/vllm_omni/diffusion/models/bagel/bagel_transformer.py @@ -714,7 +714,6 @@ class Bagel(torch.nn.Module): return generation_input, newlens, new_rope - @torch.no_grad def forward_cache_update_text( self, past_key_values: NaiveCache, @@ -821,7 +820,6 @@ class Bagel(torch.nn.Module): return generation_input, newlens, new_rope - @torch.no_grad def forward_cache_update_vae( self, vae_model, @@ -946,7 +944,6 @@ class Bagel(torch.nn.Module): return generation_input, newlens, new_rope - @torch.no_grad def forward_cache_update_vit( self, past_key_values: NaiveCache, @@ -1064,7 +1061,6 @@ class Bagel(torch.nn.Module): def prepare_vae_latent(self, curr_kvlens, curr_rope, image_sizes, new_token_ids): return self.prepare_input(curr_kvlens, curr_rope, image_sizes, new_token_ids) - @torch.no_grad def generate_image( self, packed_text_ids: torch.LongTensor, @@ -1121,7 +1117,6 @@ class Bagel(torch.nn.Module): unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) return unpacked_latent - @torch.no_grad def _forward_flow( self, x_t: torch.Tensor, diff --git a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py index ba29e681c..7d1b6578d 100644 --- a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py +++ b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py @@ -644,7 +644,6 @@ class Flux2KleinPipeline(nn.Module, SupportImageInput): def interrupt(self): return self._interrupt - @torch.no_grad() def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py b/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py index 660338a35..6bf4ca1b5 100644 --- a/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py +++ b/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py @@ -516,7 +516,6 @@ class OvisImagePipeline( def interrupt(self): return self._interrupt - @torch.no_grad() def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py index 14516a9bb..c947bd934 100644 --- a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py +++ b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py @@ -347,7 +347,6 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput): latents = latents * self.scheduler.init_noise_sigma return latents - @torch.no_grad() def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index 8ff6271a0..b691cfedb 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -278,7 +278,6 @@ class Wan22Pipeline(nn.Module): def current_timestep(self): return self._current_timestep - @torch.no_grad() def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index 635da11e6..ed8c0a987 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -254,7 +254,6 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): image_embeds = self.image_encoder(pixel_values, output_hidden_states=True) return image_embeds.hidden_states[-2] - @torch.no_grad() def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py index 6c1e39242..b943000cf 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py @@ -193,7 +193,6 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): def current_timestep(self): return self._current_timestep - @torch.no_grad() def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/diffusion/models/z_image/pipeline_z_image.py b/vllm_omni/diffusion/models/z_image/pipeline_z_image.py index 7d6fb901d..b92d3de88 100644 --- a/vllm_omni/diffusion/models/z_image/pipeline_z_image.py +++ b/vllm_omni/diffusion/models/z_image/pipeline_z_image.py @@ -312,7 +312,6 @@ class ZImagePipeline(nn.Module): def interrupt(self): return self._interrupt - @torch.no_grad() def forward( self, req: OmniDiffusionRequest, diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py index 1bc68f2a4..c60015c0e 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py @@ -1258,7 +1258,6 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel): return output - @torch.no_grad() def sample( self, conditioning_vector, @@ -1333,7 +1332,6 @@ class Qwen2_5OmniToken2WavDiTModel(Qwen2_5OmniPreTrainedModel): generated_mel_spectrogram = generated_waveform.permute(0, 2, 1) return generated_mel_spectrogram - @torch.no_grad() def fast_block_sample( self, conditioning_vector: torch.Tensor, From d2b3a24af32e0645226c7b82a1c50038f00d8740 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Tue, 27 Jan 2026 20:56:02 +0800 Subject: [PATCH 31/83] [BugFix] Modify the method of obtaining external_request_id (#961) Signed-off-by: amy-why-3459 --- tests/entrypoints/test_omni_new_request_data.py | 2 ++ vllm_omni/core/sched/omni_ar_scheduler.py | 1 + vllm_omni/core/sched/omni_generation_scheduler.py | 1 + vllm_omni/core/sched/output.py | 3 +++ vllm_omni/distributed/omni_connectors/adapter.py | 9 +++++---- .../omni_connectors/connectors/shm_connector.py | 1 + vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py | 2 -- vllm_omni/request.py | 6 ++++++ 8 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/entrypoints/test_omni_new_request_data.py b/tests/entrypoints/test_omni_new_request_data.py index 81992fb39..776509d5b 100644 --- a/tests/entrypoints/test_omni_new_request_data.py +++ b/tests/entrypoints/test_omni_new_request_data.py @@ -13,6 +13,7 @@ def test_omni_new_request_data_copies_payloads(): } request = SimpleNamespace( request_id="req-1", + external_req_id="ext-1", prompt_token_ids=[101, 102], mm_features=None, sampling_params=None, @@ -33,6 +34,7 @@ def test_omni_new_request_data_copies_payloads(): def test_omni_new_request_data_allows_missing_payloads(): request = SimpleNamespace( request_id="req-2", + external_req_id="ext-2", prompt_token_ids=[201, 202], mm_features=None, sampling_params=None, diff --git a/vllm_omni/core/sched/omni_ar_scheduler.py b/vllm_omni/core/sched/omni_ar_scheduler.py index 03eb5ff21..647eb4867 100644 --- a/vllm_omni/core/sched/omni_ar_scheduler.py +++ b/vllm_omni/core/sched/omni_ar_scheduler.py @@ -166,6 +166,7 @@ class OmniARScheduler(VLLMScheduler): # Build omni entry preserving all base fields omni_nr = OmniNewRequestData( req_id=nr.req_id, + external_req_id=(getattr(request, "external_req_id", None) if request else None), prompt_token_ids=nr.prompt_token_ids, mm_features=nr.mm_features, sampling_params=nr.sampling_params, diff --git a/vllm_omni/core/sched/omni_generation_scheduler.py b/vllm_omni/core/sched/omni_generation_scheduler.py index 0164029d2..5974adb52 100644 --- a/vllm_omni/core/sched/omni_generation_scheduler.py +++ b/vllm_omni/core/sched/omni_generation_scheduler.py @@ -210,6 +210,7 @@ class OmniGenerationScheduler(VLLMScheduler): # Build omni entry preserving all base fields omni_nr = OmniNewRequestData( req_id=nr.req_id, + external_req_id=(getattr(request, "external_req_id", None) if request else None), prompt_token_ids=nr.prompt_token_ids, mm_features=nr.mm_features, sampling_params=nr.sampling_params, diff --git a/vllm_omni/core/sched/output.py b/vllm_omni/core/sched/output.py index 3176ec14a..f4acb034d 100644 --- a/vllm_omni/core/sched/output.py +++ b/vllm_omni/core/sched/output.py @@ -21,6 +21,8 @@ class OmniNewRequestData(NewRequestData): # Optional serialized prompt embeddings prompt_embeds: PromptEmbedsPayload | None = None + # Optional external request ID for tracking + external_req_id: str | None = None # Optional serialized additional information additional_information: AdditionalInformationPayload | None = None @@ -42,6 +44,7 @@ class OmniNewRequestData(NewRequestData): """ return cls( req_id=request.request_id, + external_req_id=request.external_req_id, prompt_token_ids=request.prompt_token_ids, mm_features=request.mm_features, sampling_params=request.sampling_params, diff --git a/vllm_omni/distributed/omni_connectors/adapter.py b/vllm_omni/distributed/omni_connectors/adapter.py index da3abf4a9..a634595ac 100644 --- a/vllm_omni/distributed/omni_connectors/adapter.py +++ b/vllm_omni/distributed/omni_connectors/adapter.py @@ -199,7 +199,8 @@ def get_chunk( target_stage_id = stage_id - 1 # Handle new requests for new_req_data in scheduler_output.scheduled_new_reqs: - req_id = new_req_data.req_id[0:25] + connector.request_ids_mapping[new_req_data.req_id] = new_req_data.external_req_id + req_id = new_req_data.external_req_id chunk_id = connector.get_requests[req_id] connector_get_key = f"{req_id}_{target_stage_id}_{chunk_id}" payload_data = get_through_connector(connector, target_stage_id, stage_id, req_id, connector_get_key) @@ -214,7 +215,7 @@ def get_chunk( cached_reqs.additional_information = {} for i, cached_req_id in enumerate(cached_reqs.req_ids): - req_id = cached_req_id[0:25] + req_id = connector.request_ids_mapping.get(cached_req_id, cached_req_id) if req_id in connector.finished_requests: continue chunk_id = connector.get_requests[req_id] @@ -265,7 +266,7 @@ def get_chunk_for_generation( """ stage_id = connector.stage_id target_stage_id = stage_id - 1 - request_id = request.request_id[0:25] + request_id = request.external_req_id if request_id in connector.finished_requests: return @@ -306,7 +307,7 @@ def put_chunk( """ stage_id = connector.stage_id next_stage_id = stage_id + 1 - request_id = request.request_id[0:25] + request_id = request.external_req_id prompt_token_ids = request.prompt_token_ids connector.request_prompt_token_ids[request_id] = prompt_token_ids chunk_id = connector.put_requests[request_id] diff --git a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py index a9587e21c..bd6ec8551 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py @@ -29,6 +29,7 @@ class SharedMemoryConnector(OmniConnectorBase): self.request_payload = {} self.request_prompt_token_ids: dict[str, list[int]] = defaultdict(list) self.code_prompt_token_ids: dict[str, list[list[int]]] = defaultdict(list) + self.request_ids_mapping: dict[str, str] = {} # Default threshold matches legacy behavior (64KB) self.threshold = int(config.get("shm_threshold_bytes", 65536)) self._metrics = { diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py index 0dd590710..a42bd8e76 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py @@ -180,8 +180,6 @@ class Qwen3OmniMoeForConditionalGeneration( self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None ) - self.chunk_segment_info = {} # request_id -> chunk_segment_info - # ==================== Device utilities ==================== @staticmethod diff --git a/vllm_omni/request.py b/vllm_omni/request.py index ca5bd437d..864d9dce4 100644 --- a/vllm_omni/request.py +++ b/vllm_omni/request.py @@ -26,6 +26,8 @@ class OmniRequest(Request): def __init__( self, prompt_embeds: PromptEmbedsPayload | None = None, + # Optional external request ID for tracking + external_req_id: str | None = None, additional_information: AdditionalInformationPayload | None = None, *args, **kwargs, @@ -33,6 +35,8 @@ class OmniRequest(Request): super().__init__(*args, **kwargs) # Serialized prompt embeddings payload (optional) self.prompt_embeds: PromptEmbedsPayload | None = prompt_embeds + # Optional external request ID for tracking + self.external_req_id: str | None = external_req_id # Serialized additional information payload (optional) self.additional_information: AdditionalInformationPayload | None = additional_information @@ -54,6 +58,8 @@ class OmniRequest(Request): """ return cls( request_id=request.request_id, + # Optional external request ID for tracking + external_req_id=request.external_req_id, client_index=request.client_index, prompt_token_ids=request.prompt_token_ids, prompt_embeds=request.prompt_embeds, From 9bca8880fef1607fe16e20ad06124e292dc739d0 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:19:51 -0500 Subject: [PATCH 32/83] Fix TTS speaker typo and add supported languages (#990) Signed-off-by: linyueqian --- pyproject.toml | 1 + .../openai_api/test_serving_speech.py | 2 +- vllm_omni/entrypoints/openai/serving_speech.py | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 279d8e7d3..306733063 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,4 +200,5 @@ extend-ignore-identifiers-re = [ ".*nothink.*", ".*NOTHINK.*", ".*nin.*", + "Ono_Anna", ] diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 82b5a4fef..675af4141 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -308,7 +308,7 @@ class TestTTSMethods: assert speech_server._validate_tts_request(req) == "Input text cannot be empty" # Invalid language - req = OpenAICreateSpeechRequest(input="Hello", language="French") + req = OpenAICreateSpeechRequest(input="Hello", language="InvalidLang") assert "Invalid language" in speech_server._validate_tts_request(req) # Invalid speaker diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index d5bc20cde..223557aab 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -27,10 +27,22 @@ _TTS_SPEAKERS: set[str] = { "Eric", "Ryan", "Aiden", - "One_Anna", + "Ono_Anna", "Sohee", } -_TTS_LANGUAGES: set[str] = {"Auto", "Chinese", "English", "Japanese", "Korean"} +_TTS_LANGUAGES: set[str] = { + "Auto", + "Chinese", + "English", + "Japanese", + "Korean", + "German", + "French", + "Russian", + "Portuguese", + "Spanish", + "Italian", +} _TTS_MAX_INSTRUCTIONS_LENGTH = 500 _TTS_MAX_NEW_TOKENS_MIN = 1 _TTS_MAX_NEW_TOKENS_MAX = 4096 From 40cbd8ef7ff2d04dcf2ab0672b7e19a56303bfa9 Mon Sep 17 00:00:00 2001 From: dongbo910220 <32610838+dongbo910220@users.noreply.github.com> Date: Tue, 27 Jan 2026 23:55:05 +0800 Subject: [PATCH 33/83] [Fix] make images LoRA e2e less flaky (#978) Signed-off-by: dongbo910220 <1275604947@qq.com> Co-authored-by: Hongsheng Liu --- .buildkite/pipeline.yml | 17 +++++++++++++++++ .../test_images_generations_lora.py | 10 +++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index dcc9b3064..d1e71aba2 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -41,6 +41,23 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion Images API LoRA E2E" + timeout_in_minutes: 20 + depends_on: image-build + commands: + - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py + agents: + queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" + - label: "Diffusion Model CPU offloading Test" timeout_in_minutes: 20 depends_on: image-build diff --git a/tests/e2e/online_serving/test_images_generations_lora.py b/tests/e2e/online_serving/test_images_generations_lora.py index e5fabcb81..82044cd52 100644 --- a/tests/e2e/online_serving/test_images_generations_lora.py +++ b/tests/e2e/online_serving/test_images_generations_lora.py @@ -224,11 +224,11 @@ def test_images_generations_per_request_lora_switching(omni_server: OmniServer, base_img = _post_images(omni_server, _basic_payload()) base_slice = _image_blue_tail_slice(base_img) - # Adapter A: apply delta to Q slice only. + # Adapter A: apply delta to V slice only. lora_a_dir = tmp_path / "zimage_lora_a" - _write_zimage_lora(lora_a_dir, q_scale=0.1) + _write_zimage_lora(lora_a_dir, v_scale=8.0) payload_a = _basic_payload() - payload_a["lora"] = {"name": "a", "path": str(lora_a_dir), "scale": 2.0} + payload_a["lora"] = {"name": "a", "path": str(lora_a_dir), "scale": 64.0} img_a = _post_images(omni_server, payload_a) a_slice = _image_blue_tail_slice(img_a) _assert_slice_diff(a_slice, base_slice, label="lora_a_vs_base") @@ -236,9 +236,9 @@ def test_images_generations_per_request_lora_switching(omni_server: OmniServer, # Adapter B: apply delta to K slice only (should differ from adapter A). lora_b_dir = tmp_path / "zimage_lora_b" - _write_zimage_lora(lora_b_dir, k_scale=0.1) + _write_zimage_lora(lora_b_dir, k_scale=4.0) payload_b = _basic_payload() - payload_b["lora"] = {"name": "b", "path": str(lora_b_dir), "scale": 2.0} + payload_b["lora"] = {"name": "b", "path": str(lora_b_dir), "scale": 64.0} img_b = _post_images(omni_server, payload_b) b_slice = _image_blue_tail_slice(img_b) _assert_slice_diff(b_slice, base_slice, label="lora_b_vs_base") From e8aa32b82b6db09b04e123502fa0a8f0e8051a5a Mon Sep 17 00:00:00 2001 From: dongbo910220 <32610838+dongbo910220@users.noreply.github.com> Date: Wed, 28 Jan 2026 07:55:17 +0800 Subject: [PATCH 34/83] [Docs] Fix GLM-Image docstring indentation to resolve CI failure (#992) Signed-off-by: dongbo910220 <1275604947@qq.com> --- .../diffusion/models/glm_image/glm_image_transformer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py b/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py index c85c0bfd9..69475181d 100644 --- a/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py +++ b/vllm_omni/diffusion/models/glm_image/glm_image_transformer.py @@ -546,9 +546,9 @@ class GlmImageTransformer2DModel(CachedTransformer): This is the vllm-omni optimized version of the GLM-Image DiT model. Args: - od_config: OmniDiffusionConfig containing model configuration. The - transformer hyper-parameters (e.g. patch size / channels / heads) - are read from `od_config.tf_model_config`. + od_config: OmniDiffusionConfig containing model configuration. + Transformer hyper-parameters (e.g. patch size / channels / heads) are read from + `od_config.tf_model_config`. """ packed_modules_mapping = { From 1c8fcf436f596699d6c96c1d293feb55a3e3dc0c Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Wed, 28 Jan 2026 10:41:33 +0800 Subject: [PATCH 35/83] [Perf] replace torch's conv with vLLM's conv to fix torch 2.9 performance regression (#982) Signed-off-by: zjy0516 Co-authored-by: Hongsheng Liu --- vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py index deb626a97..e4ba6318b 100644 --- a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py +++ b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py @@ -12,6 +12,7 @@ from diffusers.models.embeddings import PixArtAlphaTextProjection, TimestepEmbed from diffusers.models.modeling_outputs import Transformer2DModelOutput from diffusers.models.normalization import FP32LayerNorm from vllm.logger import init_logger +from vllm.model_executor.layers.conv import Conv3dLayer from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import QKVParallelLinear, ReplicatedLinear from vllm.model_executor.model_loader.weight_utils import default_weight_loader @@ -605,7 +606,12 @@ class WanTransformer3DModel(nn.Module): # 1. Patch & position embedding self.rope = WanRotaryPosEmbed(attention_head_dim, patch_size, rope_max_seq_len) - self.patch_embedding = nn.Conv3d(in_channels, inner_dim, kernel_size=patch_size, stride=patch_size) + self.patch_embedding = Conv3dLayer( + in_channels=in_channels, + out_channels=inner_dim, + kernel_size=patch_size, + stride=patch_size, + ) # 2. Condition embeddings self.condition_embedder = WanTimeTextImageEmbedding( From c4220f0979cb07df4443e2688392171fd53f150f Mon Sep 17 00:00:00 2001 From: lif <1835304752@qq.com> Date: Wed, 28 Jan 2026 13:11:15 +0800 Subject: [PATCH 36/83] [BugFix] Add /health and /v1/models endpoints for diffusion mode, fix streaming compatibility (#454) Signed-off-by: majiayu000 <1835304752@qq.com> --- .../openai_api/test_image_server.py | 59 +++++- tests/test_outputs.py | 171 ++++++++++++++++++ vllm_omni/entrypoints/openai/api_server.py | 88 ++++++++- vllm_omni/outputs.py | 52 ++++++ 4 files changed, 358 insertions(+), 12 deletions(-) create mode 100644 tests/test_outputs.py diff --git a/tests/entrypoints/openai_api/test_image_server.py b/tests/entrypoints/openai_api/test_image_server.py index 45f4b33ad..86331e085 100644 --- a/tests/entrypoints/openai_api/test_image_server.py +++ b/tests/entrypoints/openai_api/test_image_server.py @@ -124,6 +124,8 @@ class FakeAsyncOmni: def mock_async_diffusion(): """Mock AsyncOmniDiffusion instance that returns fake images""" mock = Mock() + mock.is_running = True # For health endpoint + mock.check_health = AsyncMock() # For LLM mode health check async def generate(**kwargs): # Return n PIL images wrapped in result object @@ -147,8 +149,9 @@ def test_client(mock_async_diffusion): # Set up app state with diffusion engine app.state.engine_client = mock_async_diffusion + app.state.diffusion_engine = mock_async_diffusion # Also set for health endpoint app.state.stage_configs = [{"stage_type": "diffusion"}] - app.state.served_model_names = "Qwen/Qwen-Image" + app.state.diffusion_model_name = "Qwen/Qwen-Image" # For models endpoint return TestClient(app) @@ -169,10 +172,58 @@ def async_omni_test_client(): return TestClient(app) -@pytest.mark.skip(reason="Async API server uses different health check mechanism") def test_health_endpoint(test_client): - """Test health check endpoint - skipped for async server""" - pass + """Test health check endpoint for diffusion mode""" + response = test_client.get("/health") + assert response.status_code == 200 + data = response.json() + assert data["status"] == "healthy" + + +def test_health_endpoint_no_engine(): + """Test health check endpoint when no engine is initialized""" + from fastapi import FastAPI + + from vllm_omni.entrypoints.openai.api_server import router + + app = FastAPI() + app.include_router(router) + # Don't set any engine + + client = TestClient(app) + response = client.get("/health") + assert response.status_code == 503 + data = response.json() + assert data["status"] == "unhealthy" + + +def test_models_endpoint(test_client): + """Test /v1/models endpoint for diffusion mode""" + response = test_client.get("/v1/models") + assert response.status_code == 200 + data = response.json() + assert data["object"] == "list" + assert len(data["data"]) == 1 + assert data["data"][0]["id"] == "Qwen/Qwen-Image" + assert data["data"][0]["object"] == "model" + + +def test_models_endpoint_no_engine(): + """Test /v1/models endpoint when no engine is initialized""" + from fastapi import FastAPI + + from vllm_omni.entrypoints.openai.api_server import router + + app = FastAPI() + app.include_router(router) + # Don't set any engine + + client = TestClient(app) + response = client.get("/v1/models") + assert response.status_code == 200 + data = response.json() + assert data["object"] == "list" + assert len(data["data"]) == 0 def test_generate_single_image(test_client): diff --git a/tests/test_outputs.py b/tests/test_outputs.py new file mode 100644 index 000000000..c0c6201f6 --- /dev/null +++ b/tests/test_outputs.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for OmniRequestOutput class.""" + +from unittest.mock import Mock + +from PIL import Image + +from vllm_omni.outputs import OmniRequestOutput + + +class TestOmniRequestOutput: + """Tests for OmniRequestOutput class.""" + + def test_from_diffusion(self): + """Test creating output from diffusion model.""" + images = [Image.new("RGB", (64, 64), color="red")] + output = OmniRequestOutput.from_diffusion( + request_id="test-123", + images=images, + prompt="a cat", + metrics={"steps": 50}, + ) + assert output.request_id == "test-123" + assert output.images == images + assert output.prompt == "a cat" + assert output.metrics == {"steps": 50} + assert output.is_diffusion_output + assert output.num_images == 1 + + def test_from_pipeline(self): + """Test creating output from pipeline stage.""" + mock_request_output = Mock() + mock_request_output.request_id = "pipeline-123" + mock_request_output.prompt_token_ids = [1, 2, 3] + mock_request_output.outputs = [Mock()] + mock_request_output.encoder_prompt_token_ids = None + mock_request_output.prompt_logprobs = None + mock_request_output.num_cached_tokens = 10 + mock_request_output.kv_transfer_params = None + mock_request_output.multimodal_output = {"image": Mock()} + + output = OmniRequestOutput.from_pipeline( + stage_id=0, + final_output_type="text", + request_output=mock_request_output, + ) + + assert output.request_id == "pipeline-123" + assert output.stage_id == 0 + assert output.final_output_type == "text" + assert output.is_pipeline_output + + def test_prompt_token_ids_property(self): + """Test prompt_token_ids property for streaming compatibility.""" + mock_request_output = Mock() + mock_request_output.prompt_token_ids = [1, 2, 3, 4, 5] + + output = OmniRequestOutput.from_pipeline( + stage_id=0, + final_output_type="text", + request_output=mock_request_output, + ) + + assert output.prompt_token_ids == [1, 2, 3, 4, 5] + + def test_prompt_token_ids_none_when_no_request_output(self): + """Test prompt_token_ids returns None when no request_output.""" + output = OmniRequestOutput.from_diffusion( + request_id="test-123", + images=[], + prompt="a cat", + ) + assert output.prompt_token_ids is None + + def test_outputs_property(self): + """Test outputs property for chat completion compatibility.""" + mock_output = Mock() + mock_request_output = Mock() + mock_request_output.outputs = [mock_output] + + output = OmniRequestOutput.from_pipeline( + stage_id=0, + final_output_type="text", + request_output=mock_request_output, + ) + + assert output.outputs == [mock_output] + + def test_outputs_empty_when_no_request_output(self): + """Test outputs returns empty list when no request_output.""" + output = OmniRequestOutput.from_diffusion( + request_id="test-123", + images=[], + prompt="a cat", + ) + assert output.outputs == [] + + def test_encoder_prompt_token_ids_property(self): + """Test encoder_prompt_token_ids property.""" + mock_request_output = Mock() + mock_request_output.encoder_prompt_token_ids = [10, 20, 30] + + output = OmniRequestOutput.from_pipeline( + stage_id=0, + final_output_type="text", + request_output=mock_request_output, + ) + + assert output.encoder_prompt_token_ids == [10, 20, 30] + + def test_num_cached_tokens_property(self): + """Test num_cached_tokens property.""" + mock_request_output = Mock() + mock_request_output.num_cached_tokens = 42 + + output = OmniRequestOutput.from_pipeline( + stage_id=0, + final_output_type="text", + request_output=mock_request_output, + ) + + assert output.num_cached_tokens == 42 + + def test_multimodal_output_property(self): + """Test multimodal_output property.""" + mock_request_output = Mock() + mock_audio = Mock() + expected_output = {"audio": mock_audio} + mock_request_output.multimodal_output = expected_output + + output = OmniRequestOutput.from_pipeline( + stage_id=0, + final_output_type="audio", + request_output=mock_request_output, + ) + + assert output.multimodal_output is expected_output + + def test_to_dict_diffusion(self): + """Test to_dict for diffusion output.""" + output = OmniRequestOutput.from_diffusion( + request_id="test-123", + images=[Image.new("RGB", (64, 64), color="red")], + prompt="a cat", + metrics={"steps": 50}, + ) + result = output.to_dict() + + assert result["request_id"] == "test-123" + assert result["finished"] is True + assert result["final_output_type"] == "image" + assert result["num_images"] == 1 + assert result["prompt"] == "a cat" + + def test_to_dict_pipeline(self): + """Test to_dict for pipeline output.""" + mock_request_output = Mock() + mock_request_output.request_id = "pipeline-123" + + output = OmniRequestOutput.from_pipeline( + stage_id=0, + final_output_type="text", + request_output=mock_request_output, + ) + result = output.to_dict() + + assert result["request_id"] == "pipeline-123" + assert result["finished"] is True + assert result["final_output_type"] == "text" + assert result["stage_id"] == 0 diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 3c44da8b3..a47a864cd 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -666,14 +666,6 @@ def Omnispeech(request: Request) -> OmniOpenAIServingSpeech | None: return request.app.state.openai_serving_speech -# Remove the original /v1/chat/completions route before registering our own -# This prevents duplicate route registration warnings in FastAPI logs. -for route in router.routes[:]: - if hasattr(route, "path") and route.path == "/v1/chat/completions": - router.routes.remove(route) - break - - @router.post( "/v1/chat/completions", dependencies=[Depends(validate_json_request)], @@ -763,6 +755,86 @@ async def create_speech(request: OpenAICreateSpeechRequest, raw_request: Request raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e +# Health and Model endpoints for diffusion mode + + +# Remove existing health endpoint if present (from vllm imports) +# to ensure our handler takes precedence +_remove_route_from_router(router, "/health") + + +@router.get("/health") +async def health(raw_request: Request) -> JSONResponse: + """Health check endpoint that works for both LLM and diffusion modes. + + Returns 200 OK if the server is healthy. + For LLM mode: delegates to engine_client health check + For diffusion mode: checks if diffusion_engine is running + """ + # Check if we're in diffusion mode + diffusion_engine = getattr(raw_request.app.state, "diffusion_engine", None) + if diffusion_engine is not None: + # Diffusion mode health check + if hasattr(diffusion_engine, "is_running") and diffusion_engine.is_running: + return JSONResponse(content={"status": "healthy"}) + return JSONResponse( + content={"status": "unhealthy", "reason": "Diffusion engine is not running"}, + status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, + ) + + # LLM mode - delegate to engine_client + engine_client = getattr(raw_request.app.state, "engine_client", None) + if engine_client is not None: + await engine_client.check_health() + return JSONResponse(content={"status": "healthy"}) + + return JSONResponse( + content={"status": "unhealthy", "reason": "No engine initialized"}, + status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, + ) + + +# Remove existing models endpoint if present (from vllm imports) +# to ensure our handler takes precedence +_remove_route_from_router(router, "/v1/models") + + +@router.get("/v1/models") +async def show_available_models(raw_request: Request) -> JSONResponse: + """Show available models endpoint that works for both LLM and diffusion modes. + + Returns model information in OpenAI-compatible format. + """ + # Check if we're in diffusion mode + diffusion_model_name = getattr(raw_request.app.state, "diffusion_model_name", None) + if diffusion_model_name is not None: + # Diffusion mode - return the loaded model + return JSONResponse( + content={ + "object": "list", + "data": [ + { + "id": diffusion_model_name, + "object": "model", + "created": 0, + "owned_by": "vllm-omni", + "permission": [], + } + ], + } + ) + + # LLM mode - delegate to openai_serving_models + openai_serving_models = getattr(raw_request.app.state, "openai_serving_models", None) + if openai_serving_models is not None: + models = await openai_serving_models.show_available_models() + return JSONResponse(content=models.model_dump()) + + return JSONResponse( + content={"object": "list", "data": []}, + ) + + # Image generation API endpoints diff --git a/vllm_omni/outputs.py b/vllm_omni/outputs.py index 24b168287..958cebd3e 100644 --- a/vllm_omni/outputs.py +++ b/vllm_omni/outputs.py @@ -123,6 +123,58 @@ class OmniRequestOutput: """Return the number of generated images.""" return len(self.images) + # Pass-through properties keep vLLM serving codepaths compatible with + # OmniRequestOutput for pipeline outputs (Issue #345). + @property + def prompt_token_ids(self) -> list[int] | None: + """Return prompt token IDs from the underlying request output. + + This property is required for compatibility with vLLM's streaming + chat completion generator which checks res.prompt_token_ids. + """ + if self.request_output is not None: + return getattr(self.request_output, "prompt_token_ids", None) + return None + + @property + def outputs(self) -> list[Any]: + """Return outputs from the underlying request output. + + This property is required for compatibility with vLLM's streaming + and non-streaming chat completion generators. + """ + if self.request_output is not None: + return getattr(self.request_output, "outputs", []) + return [] + + @property + def encoder_prompt_token_ids(self) -> list[int] | None: + """Return encoder prompt token IDs from the underlying request output.""" + if self.request_output is not None: + return getattr(self.request_output, "encoder_prompt_token_ids", None) + return None + + @property + def prompt_logprobs(self) -> Any: + """Return prompt logprobs from the underlying request output.""" + if self.request_output is not None: + return getattr(self.request_output, "prompt_logprobs", None) + return None + + @property + def num_cached_tokens(self) -> int | None: + """Return number of cached tokens from the underlying request output.""" + if self.request_output is not None: + return getattr(self.request_output, "num_cached_tokens", None) + return None + + @property + def kv_transfer_params(self) -> Any: + """Return KV transfer params from the underlying request output.""" + if self.request_output is not None: + return getattr(self.request_output, "kv_transfer_params", None) + return None + @property def is_diffusion_output(self) -> bool: """Check if this is a diffusion model output.""" From 9ece725fc52d86ee48c0e4f4afa1bd6b0cdd8201 Mon Sep 17 00:00:00 2001 From: zhou zhuoxin Date: Wed, 28 Jan 2026 15:20:07 +0800 Subject: [PATCH 37/83] [Bugfix] Add missing dependencies (onnxruntime, sox) for Qwen3-TTS support (#981) Signed-off-by: zhou zhuoxin --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 306733063..809d1f53c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,8 @@ dependencies = [ "fa3-fwd", # flash attention 3, maintained by @ZJY0516 "openai-whisper>=20250625", "imageio[ffmpeg]>=2.37.2", + "onnxruntime>=1.19.0", + "sox>=1.5.0", # "vllm==0.14.0", # TODO: fix the entrypoints overwrite problem ] From d64bbde5616495006310a34c469baeab16d8a925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zeyu=20Huang=20=7C=20=E9=BB=83=E6=BE=A4=E5=AE=87?= <11222265+fhfuih@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:07:42 +0800 Subject: [PATCH 38/83] =?UTF-8?q?[Frontend][Model]=20Support=20batch=20req?= =?UTF-8?q?uest=20with=20refined=20OmniDiffusionReq=E2=80=A6=20(#797)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com> --- docs/getting_started/quickstart.md | 40 + .../diffusion/cache_dit_acceleration.md | 3 +- .../diffusion/parallelism_acceleration.md | 42 +- docs/user_guide/diffusion/teacache.md | 16 +- docs/user_guide/diffusion_acceleration.md | 48 +- .../offline_inference/text_to_image.md | 37 +- examples/offline_inference/bagel/end2end.py | 48 +- .../image_to_image/image_edit.py | 32 +- .../image_to_video/image_to_video.py | 25 +- .../lora_inference/lora_inference.py | 18 +- .../text_to_audio/text_to_audio.py | 23 +- .../offline_inference/text_to_image/README.md | 37 +- .../text_to_image/gradio_demo.py | 15 +- .../text_to_image/text_to_image.py | 23 +- .../text_to_video/text_to_video.py | 23 +- .../omni_connectors/test_kv_flow.py | 21 +- tests/e2e/offline_inference/conftest.py | 21 +- tests/e2e/offline_inference/test_cache_dit.py | 16 +- .../test_diffusion_cpu_offload.py | 13 +- .../offline_inference/test_diffusion_lora.py | 33 +- .../e2e/offline_inference/test_ovis_image.py | 39 +- .../test_sequence_parallel.py | 30 +- .../test_stable_audio_model.py | 23 +- tests/e2e/offline_inference/test_t2i_model.py | 15 +- tests/e2e/offline_inference/test_t2v_model.py | 18 +- tests/e2e/offline_inference/test_teacache.py | 16 +- .../test_zimage_tensor_parallel.py | 16 +- tests/e2e/online_serving/test_async_omni.py | 19 +- .../openai_api/test_image_server.py | 46 +- tests/entrypoints/test_omni_diffusion.py | 1103 +++++++++++++++++ tests/entrypoints/test_omni_llm.py | 22 +- vllm_omni/diffusion/diffusion_engine.py | 100 +- vllm_omni/diffusion/executor/abstract.py | 4 +- .../diffusion/executor/multiproc_executor.py | 6 +- .../diffusion/models/bagel/pipeline_bagel.py | 36 +- .../flux2_klein/pipeline_flux2_klein.py | 65 +- .../models/glm_image/pipeline_glm_image.py | 92 +- .../longcat_image/pipeline_longcat_image.py | 58 +- .../pipeline_longcat_image_edit.py | 105 +- .../models/ovis_image/pipeline_ovis_image.py | 35 +- .../models/qwen_image/pipeline_qwen_image.py | 37 +- .../qwen_image/pipeline_qwen_image_edit.py | 96 +- .../pipeline_qwen_image_edit_plus.py | 104 +- .../qwen_image/pipeline_qwen_image_layered.py | 112 +- .../diffusion/models/sd3/pipeline_sd3.py | 33 +- .../stable_audio/pipeline_stable_audio.py | 30 +- .../models/wan2_2/pipeline_wan2_2.py | 132 +- .../models/wan2_2/pipeline_wan2_2_i2v.py | 138 ++- .../models/wan2_2/pipeline_wan2_2_ti2v.py | 134 +- .../models/z_image/pipeline_z_image.py | 38 +- vllm_omni/diffusion/request.py | 193 +-- vllm_omni/diffusion/scheduler.py | 4 +- .../worker/gpu_diffusion_model_runner.py | 40 +- .../diffusion/worker/gpu_diffusion_worker.py | 36 +- vllm_omni/entrypoints/async_omni.py | 78 +- vllm_omni/entrypoints/async_omni_diffusion.py | 120 +- vllm_omni/entrypoints/omni.py | 132 +- vllm_omni/entrypoints/omni_diffusion.py | 63 +- vllm_omni/entrypoints/omni_stage.py | 201 ++- vllm_omni/entrypoints/openai/api_server.py | 58 +- vllm_omni/entrypoints/openai/serving_chat.py | 59 +- vllm_omni/inputs/data.py | 177 ++- .../model_executor/stage_configs/bagel.yaml | 6 - vllm_omni/outputs.py | 6 +- 64 files changed, 2995 insertions(+), 1414 deletions(-) create mode 100644 tests/entrypoints/test_omni_diffusion.py diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index b64bf1423..d4087621a 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -46,6 +46,46 @@ if __name__ == "__main__": images[0].save("coffee.png") ``` +You can pass a list of prompts and wait for them to process altogether, shown below. + +!!! info + + However, it is not currently recommended to do so + because not all models support batch inference, + and batch requesting mostly does not provide significant performance improvement (despite the impression that it does). + This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements. + +```python +from vllm_omni.entrypoints.omni import Omni + +if __name__ == "__main__": + omni = Omni( + model="Tongyi-MAI/Z-Image-Turbo", + # stage_configs_path="./stage-config.yaml", # See below + ) + prompts = [ + "a cup of coffee on a table", + "a toy dinosaur on a sandy beach", + "a fox waking up in bed and yawning", + ] + omni_outputs = omni.generate(prompts) + for i_prompt, prompt_output in enumerate(omni_outputs): + this_request_output = prompt_output.request_output[0] + this_images = this_request_output.images + for i_image, image in enumerate(this_images): + image.save(f"p{i_prompt}-img{i_image}.jpg") + print("saved to", f"p{i_prompt}-img{i_image}.jpg") + # saved to p0-img0.jpg + # saved to p1-img0.jpg + # saved to p2-img0.jpg +``` + +!!! info + + For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input + list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support + batched inputs, you can [modify this configuration](../configuration/stage_configs.md) to let the model accept a longer batch of prompts. + For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md) ## Online Serving with OpenAI-Completions API diff --git a/docs/user_guide/diffusion/cache_dit_acceleration.md b/docs/user_guide/diffusion/cache_dit_acceleration.md index 57f6a94e6..fd1bd522a 100644 --- a/docs/user_guide/diffusion/cache_dit_acceleration.md +++ b/docs/user_guide/diffusion/cache_dit_acceleration.md @@ -18,6 +18,7 @@ Enable cache-dit acceleration by simply setting `cache_backend="cache_dit"`. Cac ```python from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams # Simplest way: just enable cache-dit with default parameters omni = Omni( @@ -27,7 +28,7 @@ omni = Omni( images = omni.generate( "a beautiful landscape", - num_inference_steps=50, + OmniDiffusionSamplingParams(num_inference_steps=50), ) ``` diff --git a/docs/user_guide/diffusion/parallelism_acceleration.md b/docs/user_guide/diffusion/parallelism_acceleration.md index 09e4651ae..45e53294e 100644 --- a/docs/user_guide/diffusion/parallelism_acceleration.md +++ b/docs/user_guide/diffusion/parallelism_acceleration.md @@ -67,10 +67,12 @@ omni = Omni( ) outputs = omni.generate( - prompt="a cat reading a book", - num_inference_steps=9, - width=512, - height=512, + "a cat reading a book", + OmniDiffusionSamplingParams( + num_inference_steps=9, + width=512, + height=512, + ), ) ``` @@ -83,6 +85,7 @@ outputs = omni.generate( An example of offline inference script using [Ulysses-SP](https://arxiv.org/pdf/2309.14509) is shown below: ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.diffusion.data import DiffusionParallelConfig ulysses_degree = 2 @@ -91,7 +94,10 @@ omni = Omni( parallel_config=DiffusionParallelConfig(ulysses_degree=2) ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048), +) ``` See `examples/offline_inference/text_to_image/text_to_image.py` for a complete working example. @@ -133,6 +139,7 @@ Ring-Attention ([arxiv paper](https://arxiv.org/abs/2310.01889)) splits the inpu An example of offline inference script using Ring-Attention is shown below: ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.diffusion.data import DiffusionParallelConfig ring_degree = 2 @@ -141,7 +148,10 @@ omni = Omni( parallel_config=DiffusionParallelConfig(ring_degree=2) ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048), +) ``` See `examples/offline_inference/text_to_image/text_to_image.py` for a complete working example. @@ -183,6 +193,7 @@ You can combine both Ulysses-SP and Ring-Attention for larger scale parallelism. ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.diffusion.data import DiffusionParallelConfig # Hybrid: 2 Ulysses × 2 Ring = 4 GPUs total @@ -191,7 +202,10 @@ omni = Omni( parallel_config=DiffusionParallelConfig(ulysses_degree=2, ring_degree=2) ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048), +) ``` ##### Online Serving @@ -374,11 +388,15 @@ omni = Omni( ) outputs = omni.generate( - prompt="turn this cat to a dog", - negative_prompt="low quality, blurry", - true_cfg_scale=4.0, - pil_image=input_image, - num_inference_steps=50, + { + "prompt": "turn this cat to a dog", + "negative_prompt": "low quality, blurry", + }, + OmniDiffusionSamplingParams( + true_cfg_scale=4.0, + pil_image=input_image, + num_inference_steps=50, + ), ) ``` diff --git a/docs/user_guide/diffusion/teacache.md b/docs/user_guide/diffusion/teacache.md index 3fe961405..40dafeb88 100644 --- a/docs/user_guide/diffusion/teacache.md +++ b/docs/user_guide/diffusion/teacache.md @@ -8,6 +8,7 @@ Enable TeaCache by setting `cache_backend` to `"tea_cache"`: ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams # Simple configuration - model_type is automatically extracted from pipeline.__class__.__name__ omni = Omni( @@ -17,7 +18,12 @@ omni = Omni( "rel_l1_thresh": 0.2 # Optional, defaults to 0.2 } ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams( + num_inference_steps=50, + ), +) ``` ### Using Environment Variable @@ -68,13 +74,19 @@ Controls the balance between speed and quality. Lower values prioritize quality, ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams omni = Omni( model="Qwen/Qwen-Image", cache_backend="tea_cache", cache_config={"rel_l1_thresh": 0.2} ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams( + num_inference_steps=50, + ), +) ``` ## Performance Tuning diff --git a/docs/user_guide/diffusion_acceleration.md b/docs/user_guide/diffusion_acceleration.md index 42202d8d7..8d31747d2 100644 --- a/docs/user_guide/diffusion_acceleration.md +++ b/docs/user_guide/diffusion_acceleration.md @@ -96,6 +96,7 @@ To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** m ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams omni = Omni( model="Qwen/Qwen-Image", @@ -103,13 +104,19 @@ omni = Omni( cache_config={"rel_l1_thresh": 0.2} # Optional, defaults to 0.2 ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams( + num_inference_steps=50, + ), +) ``` ### Using Cache-DiT ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams omni = Omni( model="Qwen/Qwen-Image", @@ -123,7 +130,12 @@ omni = Omni( } ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams( + num_inference_steps=50, + ), +) ``` ### Using Ulysses-SP @@ -131,6 +143,7 @@ outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_st Run text-to-image: ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.diffusion.data import DiffusionParallelConfig ulysses_degree = 2 @@ -139,13 +152,17 @@ omni = Omni( parallel_config=DiffusionParallelConfig(ulysses_degree=ulysses_degree) ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048), +) ``` Run image-to-image: ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.diffusion.data import DiffusionParallelConfig ulysses_degree = 2 @@ -154,8 +171,13 @@ omni = Omni( parallel_config=DiffusionParallelConfig(ulysses_degree=ulysses_degree) ) -outputs = omni.generate(prompt="turn this cat to a dog", - pil_image=input_image, num_inference_steps=50) +outputs = omni.generate( + { + "prompt": "turn this cat to a dog", + "multi_modal_data": {"image": input_image} + }, + OmniDiffusionSamplingParams(num_inference_steps=50), +) ``` ### Using Ring-Attention @@ -163,6 +185,7 @@ outputs = omni.generate(prompt="turn this cat to a dog", Run text-to-image: ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.diffusion.data import DiffusionParallelConfig ring_degree = 2 @@ -171,7 +194,10 @@ omni = Omni( parallel_config=DiffusionParallelConfig(ring_degree=2) ) -outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048) +outputs = omni.generate( + "A cat sitting on a windowsill", + OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048), +) ``` ### Using CFG-Parallel @@ -182,6 +208,7 @@ CFG-Parallel splits the CFG positive/negative branches across GPUs. Use it when ```python from vllm_omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.diffusion.data import DiffusionParallelConfig cfg_parallel_size = 2 @@ -190,8 +217,13 @@ omni = Omni( parallel_config=DiffusionParallelConfig(cfg_parallel_size=cfg_parallel_size) ) -outputs = omni.generate(prompt="turn this cat to a dog", - pil_image=input_image, num_inference_steps=50, true_cfg_scale=4.0) +outputs = omni.generate( + { + "prompt": "turn this cat to a dog", + "multi_modal_data": {"image": input_image} + }, + OmniDiffusionSamplingParams(num_inference_steps=50, true_cfg_scale=4.0), +) ``` ## Documentation diff --git a/docs/user_guide/examples/offline_inference/text_to_image.md b/docs/user_guide/examples/offline_inference/text_to_image.md index 5c6f167e1..2dd0f7b7c 100644 --- a/docs/user_guide/examples/offline_inference/text_to_image.md +++ b/docs/user_guide/examples/offline_inference/text_to_image.md @@ -23,7 +23,7 @@ if __name__ == "__main__": images[0].save("coffee.png") ``` -Or put more than one prompt in a request, processing them sequentially. +Or put more than one prompt in a request. ```python from vllm_omni.entrypoints.omni import Omni @@ -40,6 +40,41 @@ if __name__ == "__main__": image = output.request_output[0].images[0].save(f"{i}.jpg") ``` +!!! info + + However, it is not currently recommended to do so + because not all models support batch inference, + and batch requesting mostly does not provide significant performance improvement (despite the impression that it does). + This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements. + +!!! info + + For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input + list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support + batched inputs, you can [modify this configuration](../../../configuration/stage_configs.md) to let the model accept a longer batch of prompts. + +Apart from string prompt, vLLM-Omni also supports dictionary prompts in the same style as vLLM. +This is useful for models that support negative prompts. + +```python +from vllm_omni.entrypoints.omni import Omni + +if __name__ == "__main__": + omni = Omni(model="Qwen/Qwen-Image") + outputs = omni.generate([ + { + "prompt": "a cup of coffee on a table"， + "negative_prompt": "low resolution" + }, + { + "prompt": "a toy dinosaur on a sandy beach"， + "negative_prompt": "cinematic, realistic" + } + ]) + for i, output in enumerate(outputs): + image = output.request_output[0].images[0].save(f"{i}.jpg") +``` + ## Local CLI Usage ```bash diff --git a/examples/offline_inference/bagel/end2end.py b/examples/offline_inference/bagel/end2end.py index 93000662d..397fd333e 100644 --- a/examples/offline_inference/bagel/end2end.py +++ b/examples/offline_inference/bagel/end2end.py @@ -1,5 +1,8 @@ import argparse import os +from typing import cast + +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType def parse_args(): @@ -50,21 +53,24 @@ def parse_args(): def main(): args = parse_args() model_name = args.model + prompts: list[OmniPromptType] = [] try: # Preferred: load from txt file (one prompt per line) if getattr(args, "txt_prompts", None) and args.prompt_type == "text": with open(args.txt_prompts, encoding="utf-8") as f: lines = [ln.strip() for ln in f.readlines()] - args.prompts = [ln for ln in lines if ln != ""] - print(f"[Info] Loaded {len(args.prompts)} prompts from {args.txt_prompts}") + prompts = [ln for ln in lines if ln != ""] + print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}") + else: + prompts = args.prompts except Exception as e: print(f"[Error] Failed to load prompts: {e}") raise - if args.prompts is None: + if not prompts: # Default prompt for text2img test if none provided - args.prompts = ["<|im_start|>A cute cat<|im_end|>"] - print(f"[Info] No prompts provided, using default: {args.prompts}") + prompts = ["<|im_start|>A cute cat<|im_end|>"] + print(f"[Info] No prompts provided, using default: {prompts}") omni_outputs = [] from PIL import Image @@ -77,21 +83,27 @@ def main(): print("[Info] Running in img2img mode (Stage 1 only)") client = OmniDiffusion(model=model_name) - generate_kwargs = { - "prompt": args.prompts, - "seed": 52, - "need_kv_receive": False, - "num_inference_steps": args.steps, - } - if args.image_path: if os.path.exists(args.image_path): loaded_image = Image.open(args.image_path).convert("RGB") - generate_kwargs["pil_image"] = loaded_image + prompts = [ + { + "prompt": cast(str, p), + "multi_modal_data": {"image": loaded_image}, + } + for p in prompts + ] else: print(f"[Warning] Image path {args.image_path} does not exist.") - result = client.generate(**generate_kwargs) + result = client.generate( + prompts, + OmniDiffusionSamplingParams( + seed=52, + need_kv_receive=False, + num_inference_steps=args.steps, + ), + ) # Ensure result is a list for iteration if not isinstance(result, list): @@ -100,8 +112,6 @@ def main(): omni_outputs = result else: - import copy - from vllm_omni.entrypoints.omni import Omni omni_kwargs = {} @@ -144,11 +154,11 @@ def main(): prompt_dict = {"prompt": final_prompt_text, "modalities": ["image"]} formatted_prompts.append(prompt_dict) - params_list = copy.deepcopy(omni.default_sampling_params_list) + params_list = omni.default_sampling_params_list if args.modality == "text2img": - params_list[0]["max_tokens"] = 1 + params_list[0].max_tokens = 1 # type: ignore # The first stage is a SamplingParam (vllm) if len(params_list) > 1: - params_list[1]["num_inference_steps"] = args.steps + params_list[1].num_inference_steps = args.steps # type: ignore # The second stage is an OmniDiffusionSamplingParam omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list)) diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index 405a0cafa..f2065e6b2 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -79,6 +79,7 @@ from PIL import Image from vllm_omni.diffusion.data import DiffusionParallelConfig, logger from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import detect_device_type, is_npu @@ -374,25 +375,28 @@ def main(): print(f"{'=' * 60}\n") generation_start = time.perf_counter() - # Generate edited image - generate_kwargs = { - "prompt": args.prompt, - "pil_image": input_image, - "negative_prompt": args.negative_prompt, - "generator": generator, - "true_cfg_scale": args.cfg_scale, - "guidance_scale": args.guidance_scale, - "num_inference_steps": args.num_inference_steps, - "num_outputs_per_prompt": args.num_outputs_per_prompt, - "layers": args.layers, - "resolution": args.resolution, - } if profiler_enabled: print("[Profiler] Starting profiling...") omni.start_profile() - outputs = omni.generate(**generate_kwargs) + # Generate edited image + outputs = omni.generate( + { + "prompt": args.prompt, + "negative_prompt": args.negative_prompt, + "multi_modal_data": {"image": input_image}, + }, + OmniDiffusionSamplingParams( + generator=generator, + true_cfg_scale=args.cfg_scale, + guidance_scale=args.guidance_scale, + num_inference_steps=args.num_inference_steps, + num_outputs_per_prompt=args.num_outputs_per_prompt, + layers=args.layers, + resolution=args.resolution, + ), + ) generation_end = time.perf_counter() generation_time = generation_end - generation_start diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 2104ec0fd..06c13fbee 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -27,6 +27,7 @@ import PIL.Image import torch from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import detect_device_type, is_npu @@ -119,16 +120,20 @@ def main(): # omni.generate() returns Generator[OmniRequestOutput, None, None] frames = omni.generate( - args.prompt, - negative_prompt=args.negative_prompt, - pil_image=image, - height=height, - width=width, - generator=generator, - guidance_scale=args.guidance_scale, - guidance_scale_2=args.guidance_scale_high, - num_inference_steps=args.num_inference_steps, - num_frames=args.num_frames, + { + "prompt": args.prompt, + "negative_prompt": args.negative_prompt, + "multi_modal_data": {"image": image}, + }, + OmniDiffusionSamplingParams( + height=height, + width=width, + generator=generator, + guidance_scale=args.guidance_scale, + guidance_scale_2=args.guidance_scale_high, + num_inference_steps=args.num_inference_steps, + num_frames=args.num_frames, + ), ) # Extract video frames from OmniRequestOutput diff --git a/examples/offline_inference/lora_inference/lora_inference.py b/examples/offline_inference/lora_inference/lora_inference.py index 17e9d6196..5e4299edb 100644 --- a/examples/offline_inference/lora_inference/lora_inference.py +++ b/examples/offline_inference/lora_inference/lora_inference.py @@ -5,6 +5,7 @@ import argparse from pathlib import Path from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.lora.request import LoRARequest from vllm_omni.lora.utils import stable_lora_int_id @@ -95,18 +96,17 @@ def main(): ) print(f"Activating pre-loaded LoRA: id={lora_request_id}, scale={args.lora_scale}") - gen_kwargs = { - "prompt": args.prompt, - "height": args.height, - "width": args.width, - "num_inference_steps": args.num_inference_steps, - } + sampling_params = OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + num_inference_steps=args.num_inference_steps, + ) if lora_request: - gen_kwargs["lora_request"] = lora_request - gen_kwargs["lora_scale"] = args.lora_scale + sampling_params.lora_request = lora_request + sampling_params.lora_scale = args.lora_scale - outputs = omni.generate(**gen_kwargs) + outputs = omni.generate(args.prompt, sampling_params) if not outputs or len(outputs) == 0: raise ValueError("No output generated from omni.generate()") diff --git a/examples/offline_inference/text_to_audio/text_to_audio.py b/examples/offline_inference/text_to_audio/text_to_audio.py index ca1a455d1..1124dbdb4 100644 --- a/examples/offline_inference/text_to_audio/text_to_audio.py +++ b/examples/offline_inference/text_to_audio/text_to_audio.py @@ -21,6 +21,7 @@ import numpy as np import torch from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.utils.platform_utils import detect_device_type @@ -143,16 +144,20 @@ def main(): # Generate audio outputs = omni.generate( - args.prompt, - negative_prompt=args.negative_prompt, - generator=generator, - guidance_scale=args.guidance_scale, - num_inference_steps=args.num_inference_steps, - num_outputs_per_prompt=args.num_waveforms, - extra={ - "audio_start_in_s": args.audio_start, - "audio_end_in_s": audio_end_in_s, + { + "prompt": args.prompt, + "negative_prompt": args.negative_prompt, }, + OmniDiffusionSamplingParams( + generator=generator, + guidance_scale=args.guidance_scale, + num_inference_steps=args.num_inference_steps, + num_outputs_per_prompt=args.num_waveforms, + extra_args={ + "audio_start_in_s": args.audio_start, + "audio_end_in_s": audio_end_in_s, + }, + ), ) generation_end = time.perf_counter() diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index 0f4351ac2..f4de19db8 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -20,7 +20,7 @@ if __name__ == "__main__": images[0].save("coffee.png") ``` -Or put more than one prompt in a request, processing them sequentially. +Or put more than one prompt in a request. ```python from vllm_omni.entrypoints.omni import Omni @@ -37,6 +37,41 @@ if __name__ == "__main__": image = output.request_output[0].images[0].save(f"{i}.jpg") ``` +!!! info + + However, it is not currently recommended to do so + because not all models support batch inference, + and batch requesting mostly does not provide significant performance improvement (despite the impression that it does). + This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements. + +!!! info + + For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input + list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support + batched inputs, you can [modify this configuration](../../../configuration/stage_configs.md) to let the model accept a longer batch of prompts. + +Apart from string prompt, vLLM-Omni also supports dictionary prompts in the same style as vLLM. +This is useful for models that support negative prompts. + +```python +from vllm_omni.entrypoints.omni import Omni + +if __name__ == "__main__": + omni = Omni(model="Qwen/Qwen-Image") + outputs = omni.generate([ + { + "prompt": "a cup of coffee on a table"， + "negative_prompt": "low resolution" + }, + { + "prompt": "a toy dinosaur on a sandy beach"， + "negative_prompt": "cinematic, realistic" + } + ]) + for i, output in enumerate(outputs): + image = output.request_output[0].images[0].save(f"{i}.jpg") +``` + ## Local CLI Usage ```bash diff --git a/examples/offline_inference/text_to_image/gradio_demo.py b/examples/offline_inference/text_to_image/gradio_demo.py index 8d4ff2b20..30446f50e 100644 --- a/examples/offline_inference/text_to_image/gradio_demo.py +++ b/examples/offline_inference/text_to_image/gradio_demo.py @@ -5,6 +5,7 @@ import gradio as gr import torch from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import detect_device_type, is_npu @@ -101,12 +102,14 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks: generator = torch.Generator(device=device).manual_seed(seed) outputs = omni.generate( prompt.strip(), - height=height, - width=width, - generator=generator, - true_cfg_scale=float(cfg_scale_value), - num_inference_steps=num_steps, - num_outputs_per_prompt=num_images, + OmniDiffusionSamplingParams( + height=height, + width=width, + generator=generator, + true_cfg_scale=float(cfg_scale_value), + num_inference_steps=num_steps, + num_outputs_per_prompt=num_images, + ), ) images_outputs = [] for output in outputs: diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 428c2437e..7fc6ec832 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -10,6 +10,7 @@ import torch from vllm_omni.diffusion.data import DiffusionParallelConfig, logger from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import detect_device_type, is_npu @@ -189,15 +190,19 @@ def main(): generation_start = time.perf_counter() outputs = omni.generate( - args.prompt, - negative_prompt=args.negative_prompt, - height=args.height, - width=args.width, - generator=generator, - true_cfg_scale=args.cfg_scale, - guidance_scale=args.guidance_scale, - num_inference_steps=args.num_inference_steps, - num_outputs_per_prompt=args.num_images_per_prompt, + { + "prompt": args.prompt, + "negative_prompt": args.negative_prompt, + }, + OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + generator=generator, + true_cfg_scale=args.cfg_scale, + guidance_scale=args.guidance_scale, + num_inference_steps=args.num_inference_steps, + num_outputs_per_prompt=args.num_images_per_prompt, + ), ) generation_end = time.perf_counter() generation_time = generation_end - generation_start diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index c655fff14..054db95b3 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -11,6 +11,7 @@ import torch from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import detect_device_type, is_npu @@ -108,15 +109,19 @@ def main(): generation_start = time.perf_counter() frames = omni.generate( - args.prompt, - negative_prompt=args.negative_prompt, - height=args.height, - width=args.width, - generator=generator, - guidance_scale=args.guidance_scale, - guidance_scale_2=args.guidance_scale_high, - num_inference_steps=args.num_inference_steps, - num_frames=args.num_frames, + { + "prompt": args.prompt, + "negative_prompt": args.negative_prompt, + }, + OmniDiffusionSamplingParams( + height=args.height, + width=args.width, + generator=generator, + guidance_scale=args.guidance_scale, + guidance_scale_2=args.guidance_scale_high, + num_inference_steps=args.num_inference_steps, + num_frames=args.num_frames, + ), ) generation_end = time.perf_counter() generation_time = generation_end - generation_start diff --git a/tests/distributed/omni_connectors/test_kv_flow.py b/tests/distributed/omni_connectors/test_kv_flow.py index a8d2f03b2..2bb06d4e0 100644 --- a/tests/distributed/omni_connectors/test_kv_flow.py +++ b/tests/distributed/omni_connectors/test_kv_flow.py @@ -1,10 +1,12 @@ import unittest +from types import SimpleNamespace from unittest.mock import MagicMock, patch import torch from vllm_omni.diffusion.models.bagel.pipeline_bagel import BagelPipeline from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner @@ -109,11 +111,12 @@ class TestKVFlow(unittest.TestCase): transfer_data.metadata = {"kv_lens": [self.seq_len], "ropes": [0]} # 2. Setup Request with Injected Data - req = OmniDiffusionRequest(prompt="test") - from types import SimpleNamespace + sp = OmniDiffusionSamplingParams( + past_key_values=SimpleNamespace(**layer_blocks), + kv_metadata=transfer_data.metadata, + ) - req.past_key_values = SimpleNamespace(**layer_blocks) - req.kv_metadata = transfer_data.metadata + req = OmniDiffusionRequest(["test"], sp) # 3. Setup Pipeline pipeline = MockBagelPipeline() @@ -144,7 +147,7 @@ class TestKVFlow(unittest.TestCase): current_cache = RealNaiveCache(self.num_layers) # --- Logic from Source Code --- - injected_kv = req.past_key_values + injected_kv = req.sampling_params.past_key_values if isinstance(current_cache, RealNaiveCache) and hasattr(injected_kv, "key_cache"): # Assuming injected_kv is SimpleNamespace or object with list attrs for layer_idx in range(len(injected_kv.key_cache)): @@ -176,9 +179,11 @@ class TestKVFlow(unittest.TestCase): data_dict = runner_test_result.to_dict() # 3. Receiver (Request Setup) - req = OmniDiffusionRequest(prompt="integration_test") - req.past_key_values = data_dict["layer_blocks"] - req.kv_metadata = data_dict["metadata"] + sp = OmniDiffusionSamplingParams( + past_key_values=data_dict["layer_blocks"], + kv_metadata=data_dict["metadata"], + ) + req = OmniDiffusionRequest(["integration_test"], sp) # noqa: F841 # 4. Receiver (Injection Simulation) # Use the logic verification again diff --git a/tests/e2e/offline_inference/conftest.py b/tests/e2e/offline_inference/conftest.py index b3a4ade92..9ee0e4b74 100644 --- a/tests/e2e/offline_inference/conftest.py +++ b/tests/e2e/offline_inference/conftest.py @@ -7,11 +7,12 @@ Pytest configuration and fixtures for vllm-omni tests. from typing import Any import pytest +from vllm import TextPrompt from vllm.distributed.parallel_state import cleanup_dist_env_and_memory -from vllm.sampling_params import SamplingParams from tests.conftest import _run_post_test_cleanup from vllm_omni.entrypoints.omni import Omni +from vllm_omni.inputs.data import OmniSamplingParams from vllm_omni.outputs import OmniRequestOutput PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None @@ -64,7 +65,7 @@ class OmniRunner: **kwargs, ) - def get_default_sampling_params_list(self) -> list[SamplingParams]: + def get_default_sampling_params_list(self) -> list[OmniSamplingParams]: """ Get a list of default sampling parameters for all stages. @@ -82,7 +83,7 @@ class OmniRunner: videos: PromptVideoInput = None, mm_processor_kwargs: dict[str, Any] | None = None, modalities: list[str] | None = None, - ) -> list[dict[str, Any]]: + ) -> list[TextPrompt]: """ Construct Omni input format from prompts and multimodal data. @@ -175,7 +176,7 @@ class OmniRunner: f"<|im_start|>assistant\n" ) - input_dict: dict[str, Any] = {"prompt": full_prompt} + input_dict: TextPrompt = {"prompt": full_prompt} if multi_modal_data: input_dict["multi_modal_data"] = multi_modal_data if modalities: @@ -189,8 +190,8 @@ class OmniRunner: def generate( self, - prompts: list[dict[str, Any]], - sampling_params_list: list[SamplingParams] | None = None, + prompts: list[TextPrompt], + sampling_params_list: list[OmniSamplingParams] | None = None, ) -> list[OmniRequestOutput]: """ Generate outputs for the given prompts. @@ -212,7 +213,7 @@ class OmniRunner: def generate_multimodal( self, prompts: list[str] | str, - sampling_params_list: list[SamplingParams] | None = None, + sampling_params_list: list[OmniSamplingParams] | None = None, system_prompt: str | None = None, audios: PromptAudioInput = None, images: PromptImageInput = None, @@ -249,7 +250,7 @@ class OmniRunner: def generate_audio( self, prompts: list[str] | str, - sampling_params_list: list[SamplingParams] | None = None, + sampling_params_list: list[OmniSamplingParams] | None = None, system_prompt: str | None = None, audios: PromptAudioInput = None, mm_processor_kwargs: dict[str, Any] | None = None, @@ -276,7 +277,7 @@ class OmniRunner: def generate_video( self, prompts: list[str] | str, - sampling_params_list: list[SamplingParams] | None = None, + sampling_params_list: list[OmniSamplingParams] | None = None, system_prompt: str | None = None, videos: PromptVideoInput = None, mm_processor_kwargs: dict[str, Any] | None = None, @@ -303,7 +304,7 @@ class OmniRunner: def generate_image( self, prompts: list[str] | str, - sampling_params_list: list[SamplingParams] | None = None, + sampling_params_list: list[OmniSamplingParams] | None = None, system_prompt: str | None = None, images: PromptImageInput = None, mm_processor_kwargs: dict[str, Any] | None = None, diff --git a/tests/e2e/offline_inference/test_cache_dit.py b/tests/e2e/offline_inference/test_cache_dit.py index 9eaee00c3..18d3988e0 100644 --- a/tests/e2e/offline_inference/test_cache_dit.py +++ b/tests/e2e/offline_inference/test_cache_dit.py @@ -15,6 +15,8 @@ from pathlib import Path import pytest import torch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: @@ -55,12 +57,14 @@ def test_cache_dit(model_name: str): outputs = m.generate( "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=num_inference_steps, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=1, # Single output for speed + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, # Single output for speed + ), ) # Extract images from request_output[0]['images'] first_output = outputs[0] diff --git a/tests/e2e/offline_inference/test_diffusion_cpu_offload.py b/tests/e2e/offline_inference/test_diffusion_cpu_offload.py index 5b43400e8..f6606dce6 100644 --- a/tests/e2e/offline_inference/test_diffusion_cpu_offload.py +++ b/tests/e2e/offline_inference/test_diffusion_cpu_offload.py @@ -6,6 +6,7 @@ import torch from vllm.distributed.parallel_state import cleanup_dist_env_and_memory from tests.utils import GPUMemoryMonitor +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.utils.platform_utils import is_npu, is_rocm # ruff: noqa: E402 @@ -30,11 +31,13 @@ def inference(model_name: str, offload: bool = True): m.generate( "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=9, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=9, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + ), ) peak = monitor.peak_used_mb monitor.stop() diff --git a/tests/e2e/offline_inference/test_diffusion_lora.py b/tests/e2e/offline_inference/test_diffusion_lora.py index 1761f253c..ef734da61 100644 --- a/tests/e2e/offline_inference/test_diffusion_lora.py +++ b/tests/e2e/offline_inference/test_diffusion_lora.py @@ -7,6 +7,7 @@ import pytest import torch from safetensors.torch import save_file +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import is_npu @@ -88,12 +89,14 @@ def test_diffusion_model(model_name: str, tmp_path: Path): outputs = m.generate( prompt, - height=height, - width=width, - num_inference_steps=2, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=1, + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=2, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, + ), ) images = _extract_images(outputs) @@ -116,14 +119,16 @@ def test_diffusion_model(model_name: str, tmp_path: Path): ) outputs_lora = m.generate( prompt, - height=height, - width=width, - num_inference_steps=2, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=1, - lora_request=lora_request, - lora_scale=2.0, + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=2, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, + lora_request=lora_request, + lora_scale=2.0, + ), ) images_lora = _extract_images(outputs_lora) assert len(images_lora) == 1 diff --git a/tests/e2e/offline_inference/test_ovis_image.py b/tests/e2e/offline_inference/test_ovis_image.py index 33e1cb880..f1bc73817 100644 --- a/tests/e2e/offline_inference/test_ovis_image.py +++ b/tests/e2e/offline_inference/test_ovis_image.py @@ -25,6 +25,7 @@ from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.models.ovis_image.pipeline_ovis_image import OvisImagePipeline from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams @pytest.fixture @@ -162,11 +163,13 @@ def test_basic_generation(ovis_pipeline): """Test the forward pass logic.""" # Setup request req = OmniDiffusionRequest( - prompt="A photo of a cat", - height=256, - width=256, - num_inference_steps=2, - guidance_scale=1.0, + prompts=["A photo of a cat"], + sampling_params=OmniDiffusionSamplingParams( + height=256, + width=256, + num_inference_steps=2, + guidance_scale=1.0, + ), ) output = ovis_pipeline(req) @@ -184,12 +187,18 @@ def test_basic_generation(ovis_pipeline): def test_guidance_scale(ovis_pipeline): """Test that classifier-free guidance path is taken when scale > 1.0.""" req = OmniDiffusionRequest( - prompt="A photo of a cat", - negative_prompt="bad quality", - height=256, - width=256, - num_inference_steps=1, - guidance_scale=2.0, # Trigger CFG + prompts=[ + { + "prompt": "A photo of a cat", + "negative_prompt": "bad quality", + } + ], + sampling_params=OmniDiffusionSamplingParams( + height=256, + width=256, + num_inference_steps=1, + guidance_scale=2.0, # Trigger CFG + ), ) ovis_pipeline(req) @@ -200,9 +209,11 @@ def test_resolution_check(ovis_pipeline): """Test resolution divisible validation logic if present.""" # Pass odd resolution req = OmniDiffusionRequest( - prompt="test", - height=250, # Not divisible by 16 (8*2) - width=250, + prompts=["test"], + sampling_params=OmniDiffusionSamplingParams( + height=250, # Not divisible by 16 (8*2) + width=250, + ), ) # Should warn but proceed (as per code I read earlier) or resize? diff --git a/tests/e2e/offline_inference/test_sequence_parallel.py b/tests/e2e/offline_inference/test_sequence_parallel.py index 9de9fd981..367e2e59e 100644 --- a/tests/e2e/offline_inference/test_sequence_parallel.py +++ b/tests/e2e/offline_inference/test_sequence_parallel.py @@ -21,6 +21,8 @@ import torch import torch.distributed as dist from PIL import Image +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[3] if str(REPO_ROOT) not in sys.path: @@ -81,12 +83,14 @@ def _run_baseline(model_name: str, dtype: torch.dtype, attn_backend: str, height try: outputs = baseline.generate( PROMPT, - height=height, - width=width, - num_inference_steps=4, - guidance_scale=0.0, - generator=torch.Generator(get_device_name()).manual_seed(seed), - num_outputs_per_prompt=1, + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=4, + guidance_scale=0.0, + generator=torch.Generator(get_device_name()).manual_seed(seed), + num_outputs_per_prompt=1, + ), ) return outputs[0].request_output[0].images finally: @@ -116,12 +120,14 @@ def _run_sp( try: outputs = sp.generate( PROMPT, - height=height, - width=width, - num_inference_steps=4, - guidance_scale=0.0, - generator=torch.Generator(get_device_name()).manual_seed(seed), - num_outputs_per_prompt=1, + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=4, + guidance_scale=0.0, + generator=torch.Generator(get_device_name()).manual_seed(seed), + num_outputs_per_prompt=1, + ), ) return outputs[0].request_output[0].images finally: diff --git a/tests/e2e/offline_inference/test_stable_audio_model.py b/tests/e2e/offline_inference/test_stable_audio_model.py index 729b501e4..df2ca5e42 100644 --- a/tests/e2e/offline_inference/test_stable_audio_model.py +++ b/tests/e2e/offline_inference/test_stable_audio_model.py @@ -5,6 +5,7 @@ import numpy as np import pytest import torch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput # ruff: noqa: E402 @@ -29,16 +30,20 @@ def test_stable_audio_model(model_name: str): sample_rate = 44100 # Stable Audio uses 44100 Hz outputs = m.generate( - "The sound of a dog barking", - negative_prompt="Low quality.", - num_inference_steps=4, # Minimal steps for speed - guidance_scale=7.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=1, - extra={ - "audio_start_in_s": audio_start_in_s, - "audio_end_in_s": audio_end_in_s, + prompts={ + "prompt": "The sound of a dog barking", + "negative_prompt": "Low quality.", }, + sampling_params_list=OmniDiffusionSamplingParams( + num_inference_steps=4, # Minimal steps for speed + guidance_scale=7.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, + extra_args={ + "audio_start_in_s": audio_start_in_s, + "audio_end_in_s": audio_end_in_s, + }, + ), ) # Extract audio from OmniRequestOutput diff --git a/tests/e2e/offline_inference/test_t2i_model.py b/tests/e2e/offline_inference/test_t2i_model.py index 09fadc93a..ba6cac34c 100644 --- a/tests/e2e/offline_inference/test_t2i_model.py +++ b/tests/e2e/offline_inference/test_t2i_model.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest import torch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils.platform_utils import is_npu, is_rocm @@ -42,12 +43,14 @@ def test_diffusion_model(model_name: str): width = 256 outputs = m.generate( "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=2, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=2, + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=2, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=2, + ), ) # Extract images from request_output[0]['images'] first_output = outputs[0] diff --git a/tests/e2e/offline_inference/test_t2v_model.py b/tests/e2e/offline_inference/test_t2v_model.py index 0263310e3..a378291ac 100644 --- a/tests/e2e/offline_inference/test_t2v_model.py +++ b/tests/e2e/offline_inference/test_t2v_model.py @@ -5,6 +5,8 @@ from pathlib import Path import pytest import torch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: @@ -33,13 +35,15 @@ def test_video_diffusion_model(model_name: str): width = 640 num_frames = 5 outputs = m.generate( - "A cat sitting on a table", - height=height, - width=width, - num_frames=num_frames, - num_inference_steps=2, - guidance_scale=1.0, - generator=torch.Generator("cuda").manual_seed(42), + prompts="A cat sitting on a table", + sampling_params_list=OmniDiffusionSamplingParams( + height=height, + width=width, + num_frames=num_frames, + num_inference_steps=2, + guidance_scale=1.0, + generator=torch.Generator("cuda").manual_seed(42), + ), ) first_output = outputs[0] assert first_output.final_output_type == "image" diff --git a/tests/e2e/offline_inference/test_teacache.py b/tests/e2e/offline_inference/test_teacache.py index 7ce915b96..7d6261388 100644 --- a/tests/e2e/offline_inference/test_teacache.py +++ b/tests/e2e/offline_inference/test_teacache.py @@ -15,6 +15,8 @@ from pathlib import Path import pytest import torch +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: @@ -51,12 +53,14 @@ def test_teacache(model_name: str): outputs = m.generate( "a photo of a cat sitting on a laptop keyboard", - height=height, - width=width, - num_inference_steps=num_inference_steps, - guidance_scale=0.0, - generator=torch.Generator("cuda").manual_seed(42), - num_outputs_per_prompt=1, # Single output for speed + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=0.0, + generator=torch.Generator("cuda").manual_seed(42), + num_outputs_per_prompt=1, # Single output for speed + ), ) # Extract images from request_output[0]['images'] first_output = outputs[0] diff --git a/tests/e2e/offline_inference/test_zimage_tensor_parallel.py b/tests/e2e/offline_inference/test_zimage_tensor_parallel.py index a9fdec4dc..ea8f0acfe 100644 --- a/tests/e2e/offline_inference/test_zimage_tensor_parallel.py +++ b/tests/e2e/offline_inference/test_zimage_tensor_parallel.py @@ -12,6 +12,8 @@ import torch from PIL import Image from vllm.distributed.parallel_state import cleanup_dist_env_and_memory +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] if str(REPO_ROOT) not in sys.path: @@ -90,12 +92,14 @@ def _run_zimage_generate( num_requests = 4 # 1 warmup + 3 timed gen = m.generate( [PROMPT] * num_requests, - height=height, - width=width, - num_inference_steps=num_inference_steps, - guidance_scale=0.0, - seed=seed, - num_outputs_per_prompt=1, + OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=0.0, + seed=seed, + num_outputs_per_prompt=1, + ), py_generator=True, ) diff --git a/tests/e2e/online_serving/test_async_omni.py b/tests/e2e/online_serving/test_async_omni.py index 112b68ff4..d90727e1b 100644 --- a/tests/e2e/online_serving/test_async_omni.py +++ b/tests/e2e/online_serving/test_async_omni.py @@ -5,6 +5,7 @@ from contextlib import ExitStack from pathlib import Path import pytest +from vllm import SamplingParams from vllm.inputs import PromptType from vllm_omni.entrypoints.async_omni import AsyncOmni, ClientRequestState @@ -25,15 +26,15 @@ async def generate( ) -> tuple[int, str]: # Ensure generate doesn't complete too fast for cancellation test. await asyncio.sleep(0.2) - thinker_sampling_params = { - "temperature": 0.4, # Deterministic - "top_p": 0.9, - "top_k": 1, - "max_tokens": max_tokens, - "repetition_penalty": 1.05, - "stop_token_ids": [151645], # Qwen EOS token <|im_end|> - "seed": SEED, - } + thinker_sampling_params = SamplingParams( + temperature=0.4, # Deterministic + top_p=0.9, + top_k=1, + max_tokens=max_tokens, + repetition_penalty=1.05, + stop_token_ids=[151645], # Qwen EOS token <|im_end|> + seed=SEED, + ) sampling_params_list = [ thinker_sampling_params, diff --git a/tests/entrypoints/openai_api/test_image_server.py b/tests/entrypoints/openai_api/test_image_server.py index 86331e085..575dfaa33 100644 --- a/tests/entrypoints/openai_api/test_image_server.py +++ b/tests/entrypoints/openai_api/test_image_server.py @@ -14,11 +14,13 @@ from unittest.mock import AsyncMock, Mock import pytest from fastapi.testclient import TestClient from PIL import Image +from vllm import SamplingParams from vllm_omni.entrypoints.openai.image_api_utils import ( encode_image_base64, parse_size, ) +from vllm_omni.inputs.data import OmniDiffusionSamplingParams # Unit Tests @@ -111,7 +113,7 @@ class FakeAsyncOmni: def __init__(self): self.stage_list = ["llm", "diffusion"] - self.default_sampling_params_list = [{"temperature": 0.1}, {"top_p": 0.9}] + self.default_sampling_params_list = [SamplingParams(temperature=0.1), OmniDiffusionSamplingParams()] self.captured_sampling_params_list = None async def generate(self, prompt, request_id, sampling_params_list): @@ -129,7 +131,8 @@ def mock_async_diffusion(): async def generate(**kwargs): # Return n PIL images wrapped in result object - n = kwargs.get("num_outputs_per_prompt", 1) + print("!!!!!!!!!!!!!!!!!!!!! kwargs", kwargs) + n = kwargs["sampling_params_list"][0].num_outputs_per_prompt images = [Image.new("RGB", (64, 64), color="blue") for _ in range(n)] return MockGenerationResult(images) @@ -268,11 +271,11 @@ def test_generate_images_async_omni_sampling_params(async_omni_test_client): captured = engine.captured_sampling_params_list assert captured is not None assert len(captured) == 2 - assert captured[0] == {"temperature": 0.1} - assert captured[1]["num_outputs_per_prompt"] == 2 - assert captured[1]["height"] == 256 - assert captured[1]["width"] == 256 - assert captured[1]["seed"] == 7 + assert captured[0].temperature == 0.1 + assert captured[1].num_outputs_per_prompt == 2 + assert captured[1].height == 256 + assert captured[1].width == 256 + assert captured[1].seed == 7 def test_generate_multiple_images(test_client): @@ -505,30 +508,11 @@ def test_parameters_passed_through(test_client, mock_async_diffusion): # Ensure generate() was called exactly once mock_async_diffusion.generate.assert_awaited_once() - call_kwargs = mock_async_diffusion.generate.call_args[1] - assert call_kwargs["num_inference_steps"] == 100 - assert call_kwargs["guidance_scale"] == 7.5 - assert call_kwargs["true_cfg_scale"] == 3.0 - assert call_kwargs["seed"] == 42 - - -def test_optional_parameters_omitted(test_client, mock_async_diffusion): - """Verify optional parameters not passed when omitted""" - response = test_client.post( - "/v1/images/generations", - json={ - "prompt": "test", - "size": "512x512", - }, - ) - assert response.status_code == 200 - - # Ensure generate() was called exactly once - mock_async_diffusion.generate.assert_awaited_once() - call_kwargs = mock_async_diffusion.generate.call_args[1] - assert "num_inference_steps" not in call_kwargs - assert "guidance_scale" not in call_kwargs - assert "true_cfg_scale" not in call_kwargs + call_kwargs = mock_async_diffusion.generate.call_args[1]["sampling_params_list"][0] + assert call_kwargs.num_inference_steps == 100 + assert call_kwargs.guidance_scale == 7.5 + assert call_kwargs.true_cfg_scale == 3.0 + assert call_kwargs.seed == 42 def test_model_field_omitted_works(test_client): diff --git a/tests/entrypoints/test_omni_diffusion.py b/tests/entrypoints/test_omni_diffusion.py new file mode 100644 index 000000000..c4884e3ab --- /dev/null +++ b/tests/entrypoints/test_omni_diffusion.py @@ -0,0 +1,1103 @@ +import uuid +import warnings +from queue import Empty, Queue +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK +from vllm_omni.inputs.data import OmniDiffusionSamplingParams + +# Suppress noisy DeprecationWarnings from optional Swig bindings imported by vLLM dependencies. +warnings.filterwarnings( + "ignore", + message=r"builtin type SwigPy.*has no __module__ attribute", + category=DeprecationWarning, +) + + +class _FakeEngineArgs(dict): + """Fake engine args that can be used both as object attributes and as **kwargs.""" + + def __init__(self, args_dict: dict[str, Any]): + super().__init__(args_dict) + # Add required attributes if not present + if "model_stage" not in self: + self["model_stage"] = None + if "engine_output_type" not in self: + self["engine_output_type"] = None + # Also set as attributes for object-style access + for key, value in self.items(): + setattr(self, key, value) + + +class _FakeStageConfig: + """Fake stage config object that mimics the real stage config structure.""" + + def __init__(self, config_dict: dict[str, Any]): + # engine_args needs to work both as object (for OmniStage) and as dict (for **kwargs) + engine_args_dict = config_dict.get("engine_args", {}) + self.engine_args = _FakeEngineArgs(engine_args_dict) + self.final_output = config_dict.get("final_output", False) + self.final_output_type = config_dict.get("final_output_type", None) + self.stage_id = config_dict.get("stage_id", 0) + # Store original dict for reference + self._config_dict = config_dict + + +class _FakeQueue: + """Fake queue using standard library Queue to replace mp.Queue.""" + + def __init__(self, maxsize=0): + self._queue = Queue(maxsize=maxsize) + + def put(self, item): + self._queue.put(item) + + def put_nowait(self, item): + self._queue.put_nowait(item) + + def get(self): + return self._queue.get() + + def get_nowait(self): + return self._queue.get_nowait() + + def empty(self): + return self._queue.empty() + + +class _FakeStage: + """Lightweight Stage stub for multi-process pipeline version with queue support.""" + + def __init__(self, config, stage_init_timeout: int = 300): + # Handle both dict and object configs + if isinstance(config, dict): + config = _FakeStageConfig(config) + self.config = config + self.stage_config = config + self.engine = None + self.engine_outputs = None + # Set attributes that OmniStage expects + self.stage_id = getattr(config, "stage_id", 0) + self.engine_args = config.engine_args + self.model_stage = getattr(config.engine_args, "model_stage", None) + self.stage_type = "diffusion" + # set default sampling params + self.default_sampling_params = OmniDiffusionSamplingParams(num_inference_steps=1) + # Allow configuring final_output and final_output_type + self.final_output = config.final_output if hasattr(config, "final_output") else False + self.final_output_type = getattr(config, "final_output_type", None) + # Configurable processing logic, default returns placeholder + processed_input = getattr(config, "_config_dict", {}).get("processed_input", ["processed"]) + self._processed_input = processed_input + # Queue references (set by attach_queues) + self._in_q = None + self._out_q = None + self._proc = None # Mock process reference + self._stage_init_timeout = max(0, int(stage_init_timeout)) + + def attach_queues(self, in_q, out_q): + """Attach input and output queues.""" + self._in_q = in_q + self._out_q = out_q + + def init_stage_worker( + self, + model: str, + *, + is_async: bool = False, + shm_threshold_bytes: int = 65536, + ctx=None, + batch_timeout: int = 10, + **kwargs, + ): + """Mock init_stage_worker: don't start real process, just send stage_ready message.""" + # Create a mock process object + self._proc = MagicMock() + self._proc.start = MagicMock() + self._proc.join = MagicMock() + self._proc.is_alive = MagicMock(return_value=False) + self._proc.terminate = MagicMock() + # Send stage_ready message to output queue + if self._out_q is not None: + try: + self._out_q.put_nowait({"type": "stage_ready", "stage_id": self.stage_id}) + except Exception: + pass + + def stop_stage_worker(self): + """Mock stop_stage_worker: clean up queue references.""" + if self._in_q is not None: + try: + self._in_q.put_nowait(SHUTDOWN_TASK) + except Exception: + pass + + def submit(self, payload: dict[str, Any]): + """Submit task to input queue.""" + if self._in_q is not None: + self._in_q.put(payload) + + def try_collect(self) -> Any: + """Non-blocking collect from output queue.""" + if self._out_q is None: + return None + try: + return self._out_q.get_nowait() + except Empty: + return None + + def set_engine_outputs(self, outputs): + """Set engine outputs for the stage.""" + self.engine_outputs = outputs + + def process_engine_inputs(self, stage_list, prompts): + """Process engine inputs: return preset processed result.""" + return self._processed_input + + +class _FakeEngine: + """Lightweight Engine stub: provides generate iterator output.""" + + def __init__(self, outputs: list[Any]): + self._outputs = outputs + + def generate(self, prompts, sampling_params): + # Record the most recent prompts for outer assertions + self._last_prompts = prompts + # Simplified: return preset list at once, ensuring iterability + yield from self._outputs + + +@pytest.fixture +def fake_stage_config(): + return { + # Don't include 'model' in engine_args since it's passed separately + "engine_args": {}, + "final_output": True, + "final_output_type": "text", + # Second stage will use processed_input to verify the chain + "processed_input": ["processed-by-stage"], + } + + +def _setup_engine_mocks(monkeypatch): + """Helper function to set up common engine mocks.""" + fake_engine = MagicMock() + # Add necessary attributes to fake_engine + fake_engine.tokenizer = MagicMock() + fake_engine.log_stats = False + fake_engine.vllm_config = MagicMock() + fake_engine.vllm_config.model_config = MagicMock() + fake_engine.vllm_config.model_config.io_processor_plugin = None + fake_engine.get_supported_tasks = MagicMock(return_value=[]) + fake_engine.model_config = MagicMock() + fake_engine.model_config.io_processor_plugin = None + # Add registry with resolve_model_cls method + fake_registry = MagicMock() + fake_registry.resolve_model_cls = MagicMock(return_value=(MagicMock(), "test_arch")) + fake_engine.model_config.registry = fake_registry + fake_engine.vllm_config.model_config.registry = fake_registry + + monkeypatch.setattr( + "vllm.v1.engine.llm_engine.LLMEngine.from_engine_args", + lambda **kw: fake_engine, + raising=False, + ) + + # Mock model_config.registry.resolve_model_cls to return a tuple + # Use a real class instead of MagicMock to avoid inspect.getsource issues + class FakeModelClass: + pass + + monkeypatch.setattr( + "vllm.model_executor.model_loader.utils.get_model_architecture", + lambda model_config: (FakeModelClass, "test_arch"), + raising=False, + ) + + monkeypatch.setattr( + "vllm.model_executor.model_loader.utils._get_model_architecture", + lambda model_config: (FakeModelClass, "test_arch"), + raising=False, + ) + + # Mock try_create_mm_pooling_model_cls to return the class as-is + monkeypatch.setattr( + "vllm.model_executor.models.adapters.try_create_mm_pooling_model_cls", + lambda model_cls: model_cls, + raising=False, + ) + + # Mock _enable_processor_cache to return False + monkeypatch.setattr( + "vllm.multimodal.cache._enable_processor_cache", + lambda model_config, mm_registry: False, + raising=False, + ) + + # Mock get_io_processor to return None + monkeypatch.setattr( + "vllm.plugins.io_processors.get_io_processor", + lambda vllm_config, io_processor_plugin: None, + raising=False, + ) + + +def _setup_multiprocessing_mocks(monkeypatch): + """Helper function to set up multiprocessing mocks.""" + import multiprocessing as mp + + # Mock Process + fake_process_class = MagicMock() + fake_process_instance = MagicMock() + fake_process_instance.start = MagicMock() + fake_process_instance.join = MagicMock() + fake_process_instance.is_alive = MagicMock(return_value=False) + fake_process_instance.terminate = MagicMock() + fake_process_class.return_value = fake_process_instance + + # Mock get_context to return a context with Queue that returns _FakeQueue + fake_ctx = MagicMock() + fake_ctx.Queue = lambda maxsize=0: _FakeQueue(maxsize=maxsize) + fake_ctx.Process = fake_process_class + + def _mock_get_context(method): + return fake_ctx + + monkeypatch.setattr(mp, "get_context", _mock_get_context, raising=False) + monkeypatch.setattr(mp, "Process", fake_process_class, raising=False) + + +def _setup_ipc_mocks(monkeypatch): + """Helper function to set up IPC function mocks.""" + + # Mock _encode: simple serialization + def _fake_encode(obj, threshold, obj_key, shm_key): + return {obj_key: obj} + + # Mock _load: extract object from result + def _fake_load(result, obj_key, shm_key): + return result.get(obj_key) + + # Mock _set: calculate serialization size + def _fake_set(obj): + return str(obj).encode() + + monkeypatch.setattr("vllm_omni.entrypoints.omni._encode", _fake_encode, raising=False) + monkeypatch.setattr("vllm_omni.entrypoints.omni._load", _fake_load, raising=False) + monkeypatch.setattr("vllm_omni.entrypoints.omni._set", _fake_set, raising=False) + + +def _setup_log_mocks(monkeypatch): + """Helper function to set up logging and stats mocks.""" + # Mock OrchestratorMetrics to be a simple class that doesn't require file operations + + class _FakeOrchestratorMetrics: + def __init__(self, num_stages, enable_stats, wall_start_ts): + self.num_stages = num_stages + self.enable_stats = enable_stats + self.stage_first_ts = [None] * num_stages + self.stage_last_ts = [None] * num_stages + self.e2e_done = set() + + def on_stage_metrics(self, stage_id, req_id, metrics): + pass + + def on_finalize_request(self, stage_id, req_id, start_ts): + self.e2e_done.add(req_id) + + def on_forward(self, from_stage, to_stage, req_id, size_bytes, tx_ms, use_shm): + pass + + def build_and_log_summary(self, final_stage_id): + return "Fake summary" + + monkeypatch.setattr( + "vllm_omni.entrypoints.omni.OrchestratorMetrics", + _FakeOrchestratorMetrics, + raising=False, + ) + + +@pytest.fixture(autouse=True) +def mock_get_config(monkeypatch): + """Auto-mock get_config and related model loading functions to avoid model path validation.""" + # CRITICAL: Mock tokenizer-related imports FIRST, before any module imports + # This prevents ImportError when async_omni is imported (which happens via omni_stage) + import sys + + fake_tokenizer = MagicMock() + fake_tokenizer.encode = MagicMock(return_value=[1, 2, 3]) + fake_tokenizer.decode = MagicMock(return_value="test") + + # Mock init_tokenizer_from_configs (used in async_omni) + def _mock_init_tokenizer_from_configs(model_config=None, **kwargs): + return fake_tokenizer + + # Strategy 1: Mock in the original location (vllm.transformers_utils.tokenizer) + # This works if the module hasn't been imported yet + monkeypatch.setattr( + "vllm.transformers_utils.tokenizer.init_tokenizer_from_configs", + _mock_init_tokenizer_from_configs, + raising=False, + ) + + # Strategy 2: If the module is already in sys.modules, patch it directly + tokenizer_module_path = "vllm.transformers_utils.tokenizer" + if tokenizer_module_path in sys.modules: + tokenizer_module = sys.modules[tokenizer_module_path] + setattr(tokenizer_module, "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs) + + # CRITICAL: Mock length_from_prompt_token_ids_or_embeds BEFORE trying to mock async_omni + + # This is because async_omni imports processor.py, which imports this function at module level + # Mock length_from_prompt_token_ids_or_embeds (used in processor.py) + def _mock_length_from_prompt_token_ids_or_embeds(prompt_token_ids=None, prompt_embeds=None): + # Return a reasonable default length + if prompt_token_ids is not None: + if isinstance(prompt_token_ids, list): + return len(prompt_token_ids) + elif hasattr(prompt_token_ids, "shape"): + return prompt_token_ids.shape[-1] if len(prompt_token_ids.shape) > 0 else 1 + if prompt_embeds is not None: + if hasattr(prompt_embeds, "shape"): + return prompt_embeds.shape[-2] if len(prompt_embeds.shape) > 1 else 1 + return 10 # Default length + + # Mock in vllm.utils + monkeypatch.setattr( + "vllm.utils.length_from_prompt_token_ids_or_embeds", + _mock_length_from_prompt_token_ids_or_embeds, + raising=False, + ) + # Also mock in processor module if it's imported + monkeypatch.setattr( + "vllm_omni.engine.input_processor.length_from_prompt_token_ids_or_embeds", + _mock_length_from_prompt_token_ids_or_embeds, + raising=False, + ) + # If processor module is already imported, patch it directly + processor_module_path = "vllm_omni.engine.input_processor" + if processor_module_path in sys.modules: + processor_module = sys.modules[processor_module_path] + setattr( + processor_module, "length_from_prompt_token_ids_or_embeds", _mock_length_from_prompt_token_ids_or_embeds + ) + + # Strategy 3: Now mock async_omni AFTER length_from_prompt_token_ids_or_embeds is mocked + # This prevents ImportError when async_omni imports processor.py + monkeypatch.setattr( + "vllm_omni.entrypoints.async_omni.init_tokenizer_from_configs", + _mock_init_tokenizer_from_configs, + raising=False, + ) + + # Strategy 4: If async_omni is already imported, patch it directly + async_omni_path = "vllm_omni.entrypoints.async_omni" + if async_omni_path in sys.modules: + async_omni_module = sys.modules[async_omni_path] + setattr(async_omni_module, "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs) + + # Now mock get_config and other functions + fake_hf_config = MagicMock() + fake_hf_config.model_type = "qwen2_5_omni" + + def _mock_get_config(model, **kwargs): + return fake_hf_config + + monkeypatch.setattr( + "vllm.transformers_utils.config.get_config", + _mock_get_config, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.get_config", + _mock_get_config, + raising=False, + ) + + # Mock transformers' cached_file to avoid downloading model configs + def _mock_cached_file(path_or_repo_id, *args, **kwargs): + import os + import tempfile + + fake_config_file = os.path.join(tempfile.gettempdir(), "fake_config.json") + if not os.path.exists(fake_config_file): + with open(fake_config_file, "w") as f: + f.write('{"model_type": "qwen2_5_omni"}') + return fake_config_file + + monkeypatch.setattr( + "transformers.utils.hub.cached_file", + _mock_cached_file, + raising=False, + ) + monkeypatch.setattr( + "transformers.utils.hub.cached_files", + lambda path_or_repo_id, filenames, **kwargs: ( + [_mock_cached_file(path_or_repo_id, filenames[0])] if filenames else None + ), + raising=False, + ) + + +def test_initialize_stage_configs_called_when_none(monkeypatch, fake_stage_config): + """Test that stage configs are auto-loaded when stage_configs_path is None.""" + + def _fake_loader(model: str, base_engine_args=None): + return [ + _FakeStageConfig(fake_stage_config), + _FakeStageConfig(fake_stage_config), + ] + + # Remove modules from cache BEFORE setting mocks + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + # Set up mocks + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + # Mock load_stage_configs_from_model + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + + # Replace OmniStage + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + # Import the module after mocks are set + import vllm_omni.entrypoints.omni as omni_module + + # Patch the imported function and class in the module + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + # Verify: auto-loaded stage_configs and stage_list have consistent count + assert isinstance(omni.stage_configs, list) + assert len(omni.stage_configs) == 2 + assert len(omni.stage_list) == 2 + # Verify: each Stage is _FakeStage instance + for st in omni.stage_list: + assert isinstance(st, _FakeStage) + # Verify: queues are attached + for st in omni.stage_list: + assert st._in_q is not None + assert st._out_q is not None + # Verify: all stages are ready + assert len(omni._stages_ready) == 2 + + +def test_generate_raises_on_length_mismatch(monkeypatch, fake_stage_config): + """Test that generate raises ValueError when sampling_params_list length doesn't match.""" + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(fake_stage_config)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + with pytest.raises(ValueError): + omni.generate(prompts=["hi"], sampling_params_list=[]) + + +def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config): + """Test multi-stage generation pipeline with queue polling.""" + stage_cfg0 = dict(fake_stage_config) + stage_cfg1 = dict(fake_stage_config) + stage_cfg1["processed_input"] = ["processed-for-stage-1"] + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(stage_cfg0), _FakeStageConfig(stage_cfg1)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + # Mock uuid.uuid4() to return a predictable value for request ID generation + test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000") + monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid) + monkeypatch.setattr(omni_module, "uuid", uuid) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + + # Generate the expected request ID format: "0_" + expected_request_id = f"0_{test_uuid}" + + # Simulate worker behavior: manually put results into output queues + # Note: We put results before calling generate, which simulates worker processes + # that have already completed. The polling loop will collect them in stage order. + # Stage 0 output (will be collected first) + omni.stage_list[0]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 0, "text": "s0"}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + # Stage 1 output (will be collected after stage 0 forwards to it) + # Note: In real flow, stage 1 result would appear after stage 0 forwards, + # but for testing we pre-populate it. The polling loop processes stages + # in order, so stage 0 result will be collected first, then forwarded, + # then stage 1 result will be collected. + omni.stage_list[1]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 1, "text": "s1"}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + + sampling_params_list = [ + OmniDiffusionSamplingParams(num_inference_steps=1), + OmniDiffusionSamplingParams(num_inference_steps=1, max_sequence_length=10), + ] + prompts = ["hi"] + outputs = omni.generate(prompts=prompts, sampling_params_list=sampling_params_list) + + # Both stages have final_output=True, so should aggregate two OmniRequestOutput + assert len(outputs) == 2 + # Verify stage outputs are set + assert omni.stage_list[0].engine_outputs == [{"stage": 0, "text": "s0"}] + assert omni.stage_list[1].engine_outputs == [{"stage": 1, "text": "s1"}] + # Verify stage 0 input queue received the task + assert not omni.stage_list[0]._in_q.empty() + # Verify stage 1 received forwarded task (process_engine_inputs was called) + assert omni.stage_list[1].process_engine_inputs([], []) is not None + + +def test_generate_pipeline_with_batch_input(monkeypatch, fake_stage_config): + """Test single-stage generation pipeline with multiple inputs in one batch.""" + stage_cfg0 = dict(fake_stage_config) + stage_cfg1 = dict(fake_stage_config) + stage_cfg0["final_output"] = False + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(stage_cfg0), _FakeStageConfig(stage_cfg1)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + # Mock uuid.uuid4() to return a predictable value for request ID generation + test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000") + monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid) + monkeypatch.setattr(omni_module, "uuid", uuid) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + + # Generate the expected request ID format: "0_" + expected_request_id = f"0_{test_uuid}" + + # Simulate worker behavior: manually put results into output queues + # Note: We put results before calling generate, which simulates worker processes + # that have already completed. The polling loop will collect them in stage order. + omni.stage_list[0]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 0, "text": "s0"}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + omni.stage_list[0]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 0, "text": "s0"}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + omni.stage_list[1]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 1}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + omni.stage_list[1]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 1}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + + outputs = omni.generate( + prompts=[ + { + "prompt": "hi", + "negative_prompt": "hi", + "multi_modal_data": {"image": ["dog.jpg", "cat.jpg"]}, + }, + { + "prompt": "hi", + "negative_prompt": "hi", + "multi_modal_data": {"image": ["dog.jpg", "cat.jpg"]}, + }, + ], + sampling_params_list=[ + OmniDiffusionSamplingParams(num_inference_steps=1), + OmniDiffusionSamplingParams(num_inference_steps=1), + ], + ) + + assert len(outputs) == 2 + + +def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config): + """Test that generate returns empty list when all stages have final_output=False.""" + stage_cfg0 = dict(fake_stage_config) + stage_cfg1 = dict(fake_stage_config) + stage_cfg0["final_output"] = False + stage_cfg1["final_output"] = False + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(stage_cfg0), _FakeStageConfig(stage_cfg1)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + # Mock uuid.uuid4() to return a predictable value for request ID generation + test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000") + monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid) + monkeypatch.setattr(omni_module, "uuid", uuid) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + + # Generate the expected request ID format: "0_" + expected_request_id = f"0_{test_uuid}" + + # Simulate worker behavior: put results into output queues + omni.stage_list[0]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 0}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + omni.stage_list[1]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 1}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + + outputs = omni.generate( + prompts=["p"], + sampling_params_list=[ + OmniDiffusionSamplingParams(num_inference_steps=1), + OmniDiffusionSamplingParams(num_inference_steps=1, max_sequence_length=10), + ], + ) + assert outputs == [] + + +def test_generate_sampling_params_none_use_default(monkeypatch, fake_stage_config): + """Test that generate uses default sampling params when sampling_params_list is None.""" + stage_cfg0 = dict(fake_stage_config) + stage_cfg1 = dict(fake_stage_config) + stage_cfg0["final_output"] = False + stage_cfg1["final_output"] = False + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(stage_cfg0), _FakeStageConfig(stage_cfg1)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + # Mock uuid.uuid4() to return a predictable value for request ID generation + test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000") + monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid) + monkeypatch.setattr(omni_module, "uuid", uuid) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + + # Generate the expected request ID format: "0_" + expected_request_id = f"0_{test_uuid}" + + # Simulate worker behavior: put results into output queues + omni.stage_list[0]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 0}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + omni.stage_list[1]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 1}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + # Use the default sampling params + omni.generate(prompts=["p"], sampling_params_list=None) + + +def test_wait_for_stages_ready_timeout(monkeypatch, fake_stage_config): + """Test that _wait_for_stages_ready handles timeout correctly.""" + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(fake_stage_config)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + + # Create a stage that doesn't send stage_ready message + class _FakeStageNoReady(_FakeStage): + def init_stage_worker(self, *args, **kwargs): + # Don't send stage_ready message + self._proc = MagicMock() + self._proc.start = MagicMock() + self._proc.join = MagicMock() + self._proc.is_alive = MagicMock(return_value=False) + self._proc.terminate = MagicMock() + + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStageNoReady(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStageNoReady(cfg, **kwargs)) + + from vllm_omni.entrypoints.omni import Omni + + # Use very short timeout + omni = Omni(model="any", init_timeout=0.01) + # Verify that no stages are ready + assert len(omni._stages_ready) == 0 + + +def test_generate_handles_error_messages(monkeypatch, fake_stage_config): + """Test that generate handles error messages from stages correctly.""" + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(fake_stage_config)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + # Mock uuid.uuid4() to return a predictable value for request ID generation + test_uuid = uuid.UUID("00000000-0000-0000-0000-000000000000") + monkeypatch.setattr(uuid, "uuid4", lambda: test_uuid) + monkeypatch.setattr(omni_module, "uuid", uuid) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + + # Generate the expected request ID format: "0_" + expected_request_id = f"0_{test_uuid}" + + # Put error message in output queue + omni.stage_list[0]._out_q.put_nowait( + { + "request_id": expected_request_id, + "error": "test error", + } + ) + # Also put a valid result after error to allow the loop to complete + # (error handling continues the loop, so we need a valid result to finish) + omni.stage_list[0]._out_q.put_nowait( + { + "request_id": expected_request_id, + "engine_outputs": [{"stage": 0, "text": "result"}], + "metrics": {"num_tokens_out": 1, "stage_gen_time_ms": 10.0}, + } + ) + + # Generate should handle error gracefully (log but continue) + sampling_params_list = [OmniDiffusionSamplingParams(num_inference_steps=1)] + outputs = omni.generate(prompts=["hi"], sampling_params_list=sampling_params_list) + # Should return final output (error was logged but didn't stop processing) + assert isinstance(outputs, list) + # Since final_output=True, should have one output + assert len(outputs) == 1 + + +def test_close_sends_shutdown_signal(monkeypatch, fake_stage_config): + """Test that close() sends shutdown signal to all input queues.""" + + def _fake_loader(model: str, base_engine_args=None): + return [_FakeStageConfig(fake_stage_config)] + + import sys + + for module_name in [ + "vllm_omni.entrypoints.utils", + "vllm_omni.entrypoints.omni", + "vllm_omni.entrypoints.omni_stage", + ]: + if module_name in sys.modules: + del sys.modules[module_name] + + _setup_engine_mocks(monkeypatch) + _setup_multiprocessing_mocks(monkeypatch) + _setup_ipc_mocks(monkeypatch) + _setup_log_mocks(monkeypatch) + + monkeypatch.setattr( + "vllm_omni.entrypoints.utils.load_stage_configs_from_model", + _fake_loader, + raising=False, + ) + monkeypatch.setattr( + "vllm_omni.entrypoints.omni_stage.OmniStage", + lambda cfg, **kwargs: _FakeStage(cfg, **kwargs), + raising=False, + ) + + import vllm_omni.entrypoints.omni as omni_module + + monkeypatch.setattr(omni_module, "load_stage_configs_from_model", _fake_loader) + monkeypatch.setattr(omni_module, "OmniStage", lambda cfg, **kwargs: _FakeStage(cfg, **kwargs)) + + from vllm_omni.entrypoints.omni import Omni + + omni = Omni(model="any", init_timeout=1) + + # Call close + omni.close() + + # Verify shutdown signal (None) was sent to input queue + # Use get_nowait to avoid blocking (close() uses put_nowait, so should be safe) + try: + shutdown_signal = omni.stage_list[0]._in_q.get_nowait() + assert shutdown_signal == SHUTDOWN_TASK + except Empty: + # If queue was already empty or only had stage_ready, that's also acceptable + # The important thing is that close() was called without error + pass + + # Verify stop_stage_worker was called (process should be set) + assert omni.stage_list[0]._proc is not None diff --git a/tests/entrypoints/test_omni_llm.py b/tests/entrypoints/test_omni_llm.py index fd2ba2673..f99c6d833 100644 --- a/tests/entrypoints/test_omni_llm.py +++ b/tests/entrypoints/test_omni_llm.py @@ -5,6 +5,7 @@ from typing import Any from unittest.mock import MagicMock import pytest +from vllm import SamplingParams from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK @@ -82,8 +83,9 @@ class _FakeStage: self.stage_id = getattr(config, "stage_id", 0) self.engine_args = config.engine_args self.model_stage = getattr(config.engine_args, "model_stage", None) + self.stage_type = "llm" # set default sampling params - self.default_sampling_params = {"temperature": 1.0} + self.default_sampling_params = SamplingParams(temperature=1.0) # Allow configuring final_output and final_output_type self.final_output = config.final_output if hasattr(config, "final_output") else False self.final_output_type = getattr(config, "final_output_type", None) @@ -637,8 +639,10 @@ def test_generate_pipeline_and_final_outputs(monkeypatch, fake_stage_config): } ) - # Use dicts instead of object() for serializable sampling params - sampling_params_list = [{"temperature": 0.7}, {"temperature": 0.8}] + sampling_params_list = [ + SamplingParams(temperature=0.7), + SamplingParams(temperature=0.8), + ] prompts = ["hi"] outputs = omni.generate(prompts=prompts, sampling_params_list=sampling_params_list) @@ -722,8 +726,13 @@ def test_generate_no_final_output_returns_empty(monkeypatch, fake_stage_config): } ) - # Use dicts instead of object() for serializable sampling params - outputs = omni.generate(prompts=["p"], sampling_params_list=[{"temperature": 0.7}, {"temperature": 0.8}]) + outputs = omni.generate( + prompts=["p"], + sampling_params_list=[ + SamplingParams(temperature=0.7), + SamplingParams(temperature=0.8), + ], + ) assert outputs == [] @@ -922,8 +931,7 @@ def test_generate_handles_error_messages(monkeypatch, fake_stage_config): ) # Generate should handle error gracefully (log but continue) - # Use dict instead of object() for serializable sampling params - sampling_params_list = [{"temperature": 0.7}] + sampling_params_list = [SamplingParams(temperature=0.7)] outputs = omni.generate(prompts=["hi"], sampling_params_list=sampling_params_list) # Should return final output (error was logged but didn't stop processing) assert isinstance(outputs, list) diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index 2e00f41b9..b0b9b89db 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -17,6 +17,7 @@ from vllm_omni.diffusion.registry import ( get_diffusion_pre_process_func, ) from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) @@ -60,36 +61,31 @@ class DiffusionEngine: self.close() raise e - def step(self, requests: list[OmniDiffusionRequest]): + def step(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]: # Apply pre-processing if available if self.pre_process_func is not None: preprocess_start_time = time.time() - requests = self.pre_process_func(requests) + request = self.pre_process_func(request) preprocess_time = time.time() - preprocess_start_time logger.info(f"Pre-processing completed in {preprocess_time:.4f} seconds") - output = self.add_req_and_wait_for_response(requests) + output = self.add_req_and_wait_for_response(request) if output.error: raise Exception(f"{output.error}") logger.info("Generation completed successfully.") if output.output is None: logger.warning("Output is None, returning empty OmniRequestOutput") - # Return empty output for the first request - if len(requests) > 0: - request = requests[0] - request_id = request.request_id or "" - prompt = request.prompt - if isinstance(prompt, list): - prompt = prompt[0] if prompt else None - return OmniRequestOutput.from_diffusion( - request_id=request_id, + return [ + OmniRequestOutput.from_diffusion( + request_id=request.request_ids[i] if i < len(request.request_ids) else "", images=[], prompt=prompt, metrics={}, latents=None, ) - return None + for i, prompt in enumerate(request.prompts) + ] postprocess_start_time = time.time() outputs = self.post_process_func(output.output) if self.post_process_func is not None else output.output @@ -102,13 +98,10 @@ class DiffusionEngine: outputs = [outputs] if outputs is not None else [] # Handle single request or multiple requests - if len(requests) == 1: + if len(request.prompts) == 1: # Single request: return single OmniRequestOutput - request = requests[0] - request_id = request.request_id or "" - prompt = request.prompt - if isinstance(prompt, list): - prompt = prompt[0] if prompt else None + prompt = request.prompts[0] + request_id = request.request_ids[0] if request.request_ids else "" metrics = {} if output.trajectory_timesteps is not None: @@ -116,37 +109,38 @@ class DiffusionEngine: if supports_audio_output(self.od_config.model_class_name): audio_payload = outputs[0] if len(outputs) == 1 else outputs - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=[], - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, - multimodal_output={"audio": audio_payload}, - final_output_type="audio", - ) + return [ + OmniRequestOutput.from_diffusion( + request_id=request_id, + images=[], + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + multimodal_output={"audio": audio_payload}, + final_output_type="audio", + ), + ] else: - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=outputs, - prompt=prompt, - metrics=metrics, - latents=output.trajectory_latents, - ) + return [ + OmniRequestOutput.from_diffusion( + request_id=request_id, + images=outputs, + prompt=prompt, + metrics=metrics, + latents=output.trajectory_latents, + ), + ] else: # Multiple requests: return list of OmniRequestOutput # Split images based on num_outputs_per_prompt for each request results = [] output_idx = 0 - for request in requests: - request_id = request.request_id or "" - prompt = request.prompt - if isinstance(prompt, list): - prompt = prompt[0] if prompt else None + for i, prompt in enumerate(request.prompts): + request_id = request.request_ids[i] if i < len(request.request_ids) else "" # Get images for this request - num_outputs = request.num_outputs_per_prompt + num_outputs = request.sampling_params.num_outputs_per_prompt request_outputs = outputs[output_idx : output_idx + num_outputs] if output_idx < len(outputs) else [] output_idx += num_outputs @@ -192,8 +186,8 @@ class DiffusionEngine: """ return DiffusionEngine(config) - def add_req_and_wait_for_response(self, requests: list[OmniDiffusionRequest]): - return self.executor.add_req(requests) + def add_req_and_wait_for_response(self, request: OmniDiffusionRequest): + return self.executor.add_req(request) def start_profile(self, trace_filename: str | None = None) -> None: """ @@ -316,8 +310,6 @@ class DiffusionEngine: def _dummy_run(self): """A dummy run to warm up the model.""" - prompt = "dummy run" - # note that num_inference_steps=1 will cause timestep and temb None in the pipeline num_inference_steps = 1 height = 1024 width = 1024 @@ -327,17 +319,19 @@ class DiffusionEngine: dummy_image = PIL.Image.new("RGB", (width, height), color=(0, 0, 0)) else: dummy_image = None + prompt: OmniTextPrompt = {"prompt": "dummy run", "multi_modal_data": {"image": dummy_image}} req = OmniDiffusionRequest( - prompt=prompt, - height=height, - width=width, - pil_image=dummy_image, - num_inference_steps=num_inference_steps, - num_outputs_per_prompt=1, + prompts=[prompt], + sampling_params=OmniDiffusionSamplingParams( + height=height, + width=width, + num_inference_steps=num_inference_steps, + num_outputs_per_prompt=1, + ), ) logger.info("dummy run to warm up the model") - requests = self.pre_process_func([req]) if self.pre_process_func is not None else [req] - self.add_req_and_wait_for_response(requests) + request = self.pre_process_func(req) if self.pre_process_func is not None else req + self.add_req_and_wait_for_response(request) def collective_rpc( self, diff --git a/vllm_omni/diffusion/executor/abstract.py b/vllm_omni/diffusion/executor/abstract.py index 021c210ef..e41f41d11 100644 --- a/vllm_omni/diffusion/executor/abstract.py +++ b/vllm_omni/diffusion/executor/abstract.py @@ -3,7 +3,7 @@ from typing import Any from vllm.utils.import_utils import resolve_obj_by_qualname -from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.request import OmniDiffusionRequest @@ -59,7 +59,7 @@ class DiffusionExecutor(ABC): pass @abstractmethod - def add_req(self, requests: list[OmniDiffusionRequest]): + def add_req(self, requests: OmniDiffusionRequest) -> DiffusionOutput: """Add requests to the execution queue.""" pass diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index a04a40e86..21a94acd3 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -6,7 +6,7 @@ from typing import Any from vllm.logger import init_logger -from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE +from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, DiffusionOutput from vllm_omni.diffusion.executor.abstract import DiffusionExecutor from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.scheduler import Scheduler @@ -130,8 +130,8 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): return processes, result_handle - def add_req(self, requests: list[OmniDiffusionRequest]): - return self.scheduler.add_req(requests) + def add_req(self, request: OmniDiffusionRequest) -> DiffusionOutput: + return self.scheduler.add_req(request) def collective_rpc( self, diff --git a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py index 5d3bcdd11..bdb9f1f5c 100644 --- a/vllm_omni/diffusion/models/bagel/pipeline_bagel.py +++ b/vllm_omni/diffusion/models/bagel/pipeline_bagel.py @@ -272,16 +272,22 @@ class BagelPipeline(nn.Module): @torch.inference_mode() def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput: - prompt = req.prompt or "" - if isinstance(prompt, list): - # vllm-omni request supports list; Bagel pipeline currently supports first prompt. - prompt = prompt[0] if prompt else "" + if len(req.prompts) > 1: + logger.warning( + """This model only supports a single prompt, not a batched request.""", + """Taking only the first image for now.""", + ) + # TODO: In online mode, sometimes it receives [{"prompts": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + first_prompt = req.prompts[0] + prompt = first_prompt if isinstance(req.prompts[0], str) else (req.prompts[0].get("prompt") or "") + max_hw = int(self.bagel.max_latent_size * self.bagel.latent_downsample) - if req.height is None and req.width is None: + if req.sampling_params.height is None and req.sampling_params.width is None: height = width = max_hw else: - height = int(req.height) if req.height is not None else max_hw - width = int(req.width) if req.width is not None else max_hw + height = int(req.sampling_params.height) if req.sampling_params.height is not None else max_hw + width = int(req.sampling_params.width) if req.sampling_params.width is not None else max_hw if height > max_hw or width > max_hw: raise ValueError( f"Requested resolution {height}x{width} exceeds Bagel checkpoint limit " @@ -292,7 +298,7 @@ class BagelPipeline(nn.Module): # Map request params to Bagel gen params (defaults follow Bagel inferencer) gen_params = BagelGenParams( - num_timesteps=int(req.num_inference_steps or 50), + num_timesteps=int(req.sampling_params.num_inference_steps or 50), timestep_shift=3.0, ) @@ -304,7 +310,7 @@ class BagelPipeline(nn.Module): # Add text prompt (prefill) on gen context. # [Omni] Check for injected KV Cache from remote transfer - injected_kv = getattr(req, "past_key_values", None) + injected_kv = req.sampling_params.past_key_values if injected_kv is not None: logger.info("Using injected KV Cache (direct)") gen_context["past_key_values"] = injected_kv @@ -316,9 +322,13 @@ class BagelPipeline(nn.Module): gen_context["ropes"] = [seq_len] else: - image_input = getattr(req, "pil_image", None) + image_input = ( + None if isinstance(first_prompt, str) else (first_prompt.get("multi_modal_data") or {}).get("image") + ) if image_input and not isinstance(image_input, list): image_input = [image_input] + if image_input: + image_input = [Image.open(image) if isinstance(image, str) else image for image in image_input] if image_input: # If we have an image, we prefill with it @@ -414,10 +424,10 @@ class BagelPipeline(nn.Module): gen_context["kv_lens"] = newlens gen_context["ropes"] = new_rope - if req.seed is not None: - torch.manual_seed(req.seed) + if req.sampling_params.seed is not None: + torch.manual_seed(req.sampling_params.seed) if self.device.type == "cuda": - torch.cuda.manual_seed(req.seed) + torch.cuda.manual_seed(req.sampling_params.seed) # Prepare latent query and run flow generation_input = self.bagel.prepare_vae_latent( diff --git a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py index 7d1b6578d..2eac7b2ec 100644 --- a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py +++ b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py @@ -19,7 +19,7 @@ import json import math import os from collections.abc import Callable, Iterable -from typing import Any +from typing import Any, cast import numpy as np import PIL.Image @@ -742,27 +742,54 @@ class Flux2KleinPipeline(nn.Module, SupportImageInput): `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated images. """ + if len(req.prompts) > 1: + logger.warning( + """This model only supports a single prompt, not a batched request.""", + """Taking only the first image for now.""", + ) + first_prompt = req.prompts[0] + prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") - prompt = req.prompt if req.prompt is not None else prompt - image = req.pil_image if req.pil_image is not None else image - height = req.height or height - width = req.width or width - num_inference_steps = req.num_inference_steps or num_inference_steps - guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale - generator = req.generator or generator - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + if ( + raw_image := None + if isinstance(first_prompt, str) + else first_prompt.get("multi_modal_data", {}).get("image") + ) is None: + pass # use image from param list + elif isinstance(raw_image, list): + image = [PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image, im) for im in raw_image] + else: + image = PIL.Image.open(raw_image) if isinstance(raw_image, str) else cast(PIL.Image.Image, raw_image) - if isinstance(req.prompt_embeds, torch.Tensor): - prompt_embeds = req.prompt_embeds - if isinstance(req.negative_prompt_embeds, torch.Tensor): - negative_prompt_embeds = req.negative_prompt_embeds + height = req.sampling_params.height or height + width = req.sampling_params.width or width + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + guidance_scale = ( + req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale + ) + generator = req.sampling_params.generator or generator + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) + max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length + text_encoder_out_layers = req.sampling_params.extra_args.get("text_encoder_out_layers", text_encoder_out_layers) - if req.max_sequence_length is not None: - max_sequence_length = req.max_sequence_length - if getattr(req, "text_encoder_out_layers", None) is not None: - text_encoder_out_layers = req.text_encoder_out_layers + req_prompt_embeds = [p.get("prompt_embeds") if not isinstance(p, str) else None for p in req.prompts] + if any(p is not None for p in req_prompt_embeds): + # If at list one prompt is provided as an embedding, + # Then assume that the user wants to provide embeddings for all prompts, and enter this if block + # If the user in fact provides mixed input format, req_prompt_embeds will have some None's + # And `torch.stack` automatically raises an exception for us + prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore # intentionally expect TypeError + + req_negative_prompt_embeds = [ + p.get("negative_prompt_embeds") if not isinstance(p, str) else None for p in req.prompts + ] + if any(p is not None for p in req_negative_prompt_embeds): + negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore # intentionally expect TypeError # 1. Check inputs. Raise error if not correct self.check_inputs( diff --git a/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py b/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py index 9a03a9349..d222342b5 100644 --- a/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py +++ b/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py @@ -17,6 +17,7 @@ import logging import os import re from collections.abc import Iterable +from typing import cast import numpy as np import PIL.Image @@ -48,6 +49,7 @@ from vllm_omni.diffusion.models.glm_image.glm_image_transformer import ( GlmImageTransformer2DModel, ) from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) @@ -77,16 +79,26 @@ def get_glm_image_pre_process_func(od_config: OmniDiffusionConfig): # GLM-Image uses patch_size=2 for transformer patch_size = 2 - def pre_process_func(requests: list[OmniDiffusionRequest]): + def pre_process_func(request: OmniDiffusionRequest): """Pre-process condition images for Image Edit mode.""" - for req in requests: - images = req.pil_image - if images is None: + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} + + if raw_image is None: # Text-to-image mode, no preprocessing needed continue - if not isinstance(images, list): - images = [images] + if not isinstance(raw_image, list): + raw_image = [raw_image] + images = [ + PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image | np.ndarray | torch.Tensor, im) + for im in raw_image + ] preprocessed = [] height, width = None, None @@ -110,14 +122,19 @@ def get_glm_image_pre_process_func(od_config: OmniDiffusionConfig): height, width = img_h, img_w # Store in request - req.preprocessed_image = preprocessed - req.prompt_image = images # Keep original PIL images - if req.height is None: - req.height = height - if req.width is None: - req.width = width + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt, additional_information={}) + elif "additional_information" not in prompt: + prompt["additional_information"] = {} + prompt["additional_information"]["preprocessed_image"] = processed # type: ignore + prompt["additional_information"]["prompt_image"] = images # type: ignore + request.prompts[i] = prompt + if request.sampling_params.height is None: + request.sampling_params.height = height + if request.sampling_params.width is None: + request.sampling_params.width = width - return requests + return request return pre_process_func @@ -821,27 +838,44 @@ class GlmImagePipeline(nn.Module): Returns: DiffusionOutput containing generated image """ - prompt = req.prompt or "" - if isinstance(prompt, list): - prompt = prompt[0] if prompt else "" + if len(req.prompts) > 1: + logger.warning( + """This model only supports a single prompt, not a batched request.""", + """Taking only the first image for now.""", + ) + first_prompt = req.prompts[0] + prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") # Get pre-computed prompt embeddings if provided - prompt_embeds = req.prompt_embeds if isinstance(req.prompt_embeds, torch.Tensor) else None + if isinstance(first_prompt, str): + prompt_embeds = None + else: + prompt_embeds = first_prompt.get("prompt_embeds") + if not isinstance(prompt_embeds, torch.Tensor): + prompt_embeds = None # Get condition images for Image Edit mode # Use pre-processed images from pre_process_func - preprocessed_images = req.preprocessed_image - condition_images = getattr(req, "prompt_image", None) - img_height = req.height - img_width = req.width + preprocessed_images = ( + None + if isinstance(first_prompt, str) + else first_prompt.get("additional_information", {}).get("preprocessed_image") + ) + condition_images = ( + None + if isinstance(first_prompt, str) + else first_prompt.get("additional_information", {}).get("prompt_image") + ) + img_height = req.sampling_params.height + img_width = req.sampling_params.width is_image_edit = preprocessed_images is not None # Use image dimensions as default if available - height = req.height or img_height or self.default_sample_size * self.vae_scale_factor - width = req.width or img_width or self.default_sample_size * self.vae_scale_factor - num_inference_steps = req.num_inference_steps or 50 - guidance_scale = req.guidance_scale or 1.5 + height = req.sampling_params.height or img_height or self.default_sample_size * self.vae_scale_factor + width = req.sampling_params.width or img_width or self.default_sample_size * self.vae_scale_factor + num_inference_steps = req.sampling_params.num_inference_steps or 50 + guidance_scale = req.sampling_params.guidance_scale or 1.5 # 0. Validate inputs self.check_inputs(prompt=prompt, height=height, width=width, prompt_embeds=prompt_embeds) @@ -851,13 +885,13 @@ class GlmImagePipeline(nn.Module): # Set seed if provided generator = None - if req.seed is not None: - generator = torch.Generator(device=self.device).manual_seed(req.seed) + if req.sampling_params.seed is not None: + generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed) # 1. Get prior tokens - either from external source (multistage) or generate internally # Check if prior_token_ids are provided externally (from AR stage in multistage mode) - external_prior_tokens = req.extra.get("prior_token_ids") if req.extra else None - external_prior_image_ids = req.extra.get("prior_token_image_ids") if req.extra else None + external_prior_tokens = req.sampling_params.extra_args.get("prior_token_ids") + external_prior_image_ids = req.sampling_params.extra_args.get("prior_token_image_ids") if external_prior_tokens is not None: # Multistage mode: use externally provided prior tokens from vLLM AR stage diff --git a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py index e918afe0b..8b616ec45 100644 --- a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py +++ b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py @@ -451,7 +451,7 @@ class LongCatImagePipeline( raise ValueError( "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." ) - elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + elif prompt is not None and not isinstance(prompt, (str, list)): raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") if negative_prompt is not None and negative_prompt_embeds is not None: @@ -470,11 +470,11 @@ class LongCatImagePipeline( num_inference_steps: int = 50, sigmas: list[float] | None = None, guidance_scale: float = 4.5, - num_images_per_prompt: int | None = 1, + num_images_per_prompt: int = 1, generator: torch.Generator | list[torch.Generator] | None = None, latents: torch.FloatTensor | None = None, - prompt_embeds: torch.FloatTensor | None = None, - negative_prompt_embeds: torch.FloatTensor | None = None, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, output_type: str | None = "pil", return_dict: bool = True, joint_attention_kwargs: dict[str, Any] | None = None, @@ -482,20 +482,44 @@ class LongCatImagePipeline( cfg_renorm_min: float | None = 0.0, enable_prompt_rewrite: bool | None = True, ) -> DiffusionOutput: - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt + if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): + negative_prompt = None + elif req.prompts: + negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] - height = req.height or height or self.default_sample_size * self.vae_scale_factor - width = req.width or width or self.default_sample_size * self.vae_scale_factor - num_inference_steps = req.num_inference_steps or num_inference_steps - generator = req.generator or generator - guidance_scale = req.guidance_scale if getattr(req, "guidance_scale", None) is not None else guidance_scale - num_images_per_prompt = getattr(req, "num_outputs_per_prompt", None) or num_images_per_prompt - enable_prompt_rewrite = getattr(req, "enable_prompt_rewrite", None) or enable_prompt_rewrite - enable_cfg_renorm = getattr(req, "enable_cfg_renorm", None) or enable_cfg_renorm - cfg_renorm_min = getattr(req, "cfg_renorm_min", None) or cfg_renorm_min - prompt_embeds = getattr(req, "prompt_embeds", None) or prompt_embeds - negative_prompt_embeds = getattr(req, "negative_prompt_embeds", None) or negative_prompt_embeds + height = req.sampling_params.height or height or self.default_sample_size * self.vae_scale_factor + width = req.sampling_params.width or width or self.default_sample_size * self.vae_scale_factor + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + generator = req.sampling_params.generator or generator + guidance_scale = ( + req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale + ) + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt is not None + else num_images_per_prompt + ) + enable_prompt_rewrite = req.sampling_params.extra_args.get("enable_prompt_rewrite", enable_prompt_rewrite) + enable_cfg_renorm = req.sampling_params.extra_args.get("enable_cfg_renorm", enable_cfg_renorm) + cfg_renorm_min = req.sampling_params.extra_args.get("cfg_renorm_min", cfg_renorm_min) + + req_prompt_embeds = [p.get("prompt_embeds") if not isinstance(p, str) else None for p in req.prompts] + if any(p is not None for p in req_prompt_embeds): + # If at list one prompt is provided as an embedding, + # Then assume that the user wants to provide embeddings for all prompts, and enter this if block + # If the user in fact provides mixed input format, req_prompt_embeds will have some None's + # And `torch.stack` automatically raises an exception for us + prompt_embeds = torch.stack(req_prompt_embeds) # type: ignore # intentionally expect TypeError + + req_negative_prompt_embeds = [ + p.get("negative_prompt_embeds") if not isinstance(p, str) else None for p in req.prompts + ] + if any(p is not None for p in req_negative_prompt_embeds): + negative_prompt_embeds = torch.stack(req_negative_prompt_embeds) # type: ignore # intentionally expect TypeError self.check_inputs( prompt, diff --git a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py index 607553aee..f2c3fd648 100644 --- a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py +++ b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py @@ -6,7 +6,7 @@ import math import os import re from collections.abc import Iterable -from typing import Any +from typing import Any, cast import numpy as np import PIL.Image @@ -34,6 +34,7 @@ from vllm_omni.diffusion.models.longcat_image.longcat_image_transformer import ( ) from vllm_omni.diffusion.models.longcat_image.pipeline_longcat_image import calculate_shift from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) @@ -59,22 +60,38 @@ def get_longcat_image_edit_pre_process_func( latent_channels = vae_config.get("latent_channels", 16) def pre_process_func( - requests: list[OmniDiffusionRequest], + request: OmniDiffusionRequest, ): - """Pre-process requests for QwenImageEditPipeline.""" - for req in requests: - image = req.pil_image + """Pre-process requests for LongCatImageEditPipeline.""" + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} - image_size = image[0].size if isinstance(image, list) else image.size + if raw_image is None or isinstance(raw_image, list): + raise ValueError( + """Received no image or a list of image. Only a single image is supported by this model.""" + """Please correctly set `"multi_modal_data": {"image": , …}`""" + ) + + if isinstance(raw_image, str): + image = PIL.Image.open(raw_image) + else: + image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image) + + image_size = image.size calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] * 1.0 / image_size[1]) - height = req.height or calculated_height - width = req.width or calculated_width + height = request.sampling_params.height or calculated_height + width = request.sampling_params.width or calculated_width # Store calculated dimensions in request - req.calculated_height = calculated_height - req.calculated_width = calculated_width - req.height = height - req.width = width + prompt["additional_information"]["calculated_height"] = calculated_height + prompt["additional_information"]["calculated_width"] = calculated_width + request.sampling_params.height = height + request.sampling_params.width = width # Preprocess image if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == latent_channels): @@ -83,9 +100,10 @@ def get_longcat_image_edit_pre_process_func( image = image_processor.preprocess(image, calculated_height, calculated_width) # Store preprocessed image and prompt image in request - req.preprocessed_image = image - req.prompt_image = prompt_image - return requests + prompt["additional_information"]["preprocessed_image"] = image + prompt["additional_information"]["prompt_image"] = prompt_image + request.prompts[i] = prompt + return request return pre_process_func @@ -500,42 +518,59 @@ class LongCatImageEditPipeline(nn.Module, SupportImageInput): self, req: OmniDiffusionRequest, image: PIL.Image.Image | torch.Tensor | None = None, - prompt: str | list[str] = None, - negative_prompt: str | list[str] = None, + prompt: str | list[str] | None = None, + negative_prompt: str | list[str] | None = None, num_inference_steps: int = 50, sigmas: list[float] | None = None, guidance_scale: float = 3.5, num_images_per_prompt: int | None = 1, generator: torch.Generator | list[torch.Generator] | None = None, latents: torch.FloatTensor | None = None, - prompt_embeds: torch.FloatTensor | None = None, - negative_prompt_embeds: torch.FloatTensor | None = None, + prompt_embeds: torch.Tensor | None = None, + negative_prompt_embeds: torch.Tensor | None = None, output_type: str | None = "pil", return_dict: bool = True, joint_attention_kwargs: dict[str, Any] | None = None, ): - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt - negative_prompt = "" if negative_prompt is None else negative_prompt - guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale - num_inference_steps = req.num_inference_steps or num_inference_steps - num_images_per_prompt = getattr(req, "num_outputs_per_prompt", None) or num_images_per_prompt - generator = req.generator or generator - prompt_embeds = getattr(req, "prompt_embeds", None) or prompt_embeds - negative_prompt_embeds = getattr(req, "negative_prompt_embeds", None) or negative_prompt_embeds - height = req.height or self.default_sample_size * self.vae_scale_factor - width = req.width or self.default_sample_size * self.vae_scale_factor + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + if len(req.prompts) > 1: + logger.warning( + """This model only supports a single prompt, not a batched request.""", + """Taking only the first image for now.""", + ) + first_prompt = req.prompts[0] + prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") + negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt") + prompt_embeds = None if isinstance(first_prompt, str) else first_prompt.get("prompt_embeds") + negative_prompt_embeds = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt_embeds") # type: ignore # Why it is list[torch.Tensor] in OmniTokenInputs or OmniEmbedsPrompt? Doesn't make sense + + sigmas = req.sampling_params.sigmas or sigmas + guidance_scale = ( + req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale + ) + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt is not None + else num_images_per_prompt + ) + generator = req.sampling_params.generator or generator + height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor + width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor if prompt is not None: batch_size = 1 if isinstance(prompt, str) else len(prompt) else: batch_size = prompt_embeds.shape[0] - if hasattr(req, "preprocessed_image"): - prompt_image = req.prompt_image - image = req.preprocessed_image - calculated_height = req.calculated_height if hasattr(req, "calculated_height") else height - calculated_width = req.calculated_width if hasattr(req, "calculated_width") else width + if not isinstance(first_prompt, str) and "preprocessed_image" in ( + additional_information := first_prompt.get("additional_information", {}) + ): + prompt_image = additional_information.get("prompt_image") + image = additional_information.get("preprocessed_image") + calculated_height = additional_information.get("calculated_height", height) + calculated_width = additional_information.get("calculated_width", width) else: image_size = image[0].size if isinstance(image, list) else image.size calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] * 1.0 / image_size[1]) diff --git a/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py b/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py index 6bf4ca1b5..a87410a6c 100644 --- a/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py +++ b/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py @@ -519,8 +519,8 @@ class OvisImagePipeline( def forward( self, req: OmniDiffusionRequest, - prompt: str | list[str] = None, - negative_prompt: str | list[str] = None, + prompt: str | list[str] | None = None, + negative_prompt: str | list[str] | None = None, guidance_scale: float = 5.0, height: int | None = None, width: int | None = None, @@ -606,16 +606,27 @@ class OvisImagePipeline( [`~pipelines.ovis_image.OvisImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated images. """ - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt - height = req.height or self.default_sample_size * self.vae_scale_factor - width = req.width or self.default_sample_size * self.vae_scale_factor - num_inference_steps = req.num_inference_steps or num_inference_steps - guidance_scale = req.guidance_scale if req.guidance_scale is not None else guidance_scale - generator = req.generator or generator - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt + if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): + negative_prompt = None + elif req.prompts: + negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] + + height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor + width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + guidance_scale = ( + req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale + ) + generator = req.sampling_params.generator or generator + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) # Steps: # 1. Check Inputs diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py index 2b9d906d3..e0a37b8bc 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py @@ -662,21 +662,28 @@ class QwenImagePipeline( callback_on_step_end_tensor_inputs: list[str] = ["latents"], max_sequence_length: int = 512, ) -> DiffusionOutput: - # # TODO: only support single prompt now - # if req.prompt is not None: - # prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt - prompt = req.prompt - negative_prompt = req.negative_prompt - height = req.height or self.default_sample_size * self.vae_scale_factor - width = req.width or self.default_sample_size * self.vae_scale_factor - num_inference_steps = req.num_inference_steps or num_inference_steps - generator = req.generator or generator - true_cfg_scale = req.true_cfg_scale or true_cfg_scale - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt + if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): + negative_prompt = None + elif req.prompts: + negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] + + height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor + width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length + generator = req.sampling_params.generator or generator + true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) # 1. check inputs # 2. encode prompts # 3. prepare latents and timesteps diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py index ac74507c7..ba65bf07e 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py @@ -7,7 +7,7 @@ import logging import math import os from collections.abc import Iterable -from typing import Any +from typing import Any, cast import numpy as np import PIL.Image @@ -39,6 +39,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( ) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs +from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) @@ -64,16 +65,33 @@ def get_qwen_image_edit_pre_process_func( latent_channels = vae_config.get("z_dim", 16) def pre_process_func( - requests: list[OmniDiffusionRequest], + request: OmniDiffusionRequest, ): """Pre-process requests for QwenImageEditPipeline.""" - for req in requests: - image = req.pil_image + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} - image_size = image[0].size if isinstance(image, list) else image.size + # Only handles single image + if raw_image is None or isinstance(raw_image, list): + raise ValueError( + """Received no image or a list of image. Only a single image is supported by this model.""" + """Please correctly set `"multi_modal_data": {"image": , …}`""" + ) + + if isinstance(raw_image, str): + image = PIL.Image.open(raw_image) + else: + image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image) + + image_size = image.size calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] / image_size[1]) - height = req.height or calculated_height - width = req.width or calculated_width + height = request.sampling_params.height or calculated_height + width = request.sampling_params.width or calculated_width # Ensure dimensions are multiples of vae_scale_factor * 2 multiple_of = vae_scale_factor * 2 @@ -81,10 +99,10 @@ def get_qwen_image_edit_pre_process_func( width = width // multiple_of * multiple_of # Store calculated dimensions in request - req.calculated_height = calculated_height - req.calculated_width = calculated_width - req.height = height - req.width = width + prompt["additional_information"]["calculated_height"] = calculated_height + prompt["additional_information"]["calculated_width"] = calculated_width + request.sampling_params.height = height + request.sampling_params.width = width # Preprocess image if image is not None and not ( @@ -96,10 +114,10 @@ def get_qwen_image_edit_pre_process_func( image = image.unsqueeze(2) # Store preprocessed image and prompt image in request - req.preprocessed_image = image - req.prompt_image = prompt_image - - return requests + prompt["additional_information"]["preprocessed_image"] = image + prompt["additional_information"]["prompt_image"] = prompt_image + request.prompts[i] = prompt + return request return pre_process_func @@ -717,17 +735,27 @@ class QwenImageEditPipeline( max_sequence_length: int = 512, ) -> DiffusionOutput: """Forward pass for image editing.""" - prompt = req.prompt - negative_prompt = req.negative_prompt + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + if len(req.prompts) > 1: + logger.warning( + """This model only supports a single prompt, not a batched request.""", + """Taking only the first image for now.""", + ) + first_prompt = req.prompts[0] + prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") + negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt") # Get preprocessed image from request (pre-processing is done in DiffusionEngine) - if hasattr(req, "preprocessed_image"): - prompt_image = req.prompt_image - image = req.preprocessed_image - calculated_height = req.calculated_height - calculated_width = req.calculated_width - height = req.height - width = req.width + if not isinstance(first_prompt, str) and "preprocessed_image" in ( + additional_information := first_prompt.get("additional_information", {}) + ): + prompt_image = additional_information.get("prompt_image") + image = additional_information.get("preprocessed_image") + calculated_height = additional_information.get("calculated_height") + calculated_width = additional_information.get("calculated_width") + height = req.sampling_params.height + width = req.sampling_params.width else: # fallback to run pre-processing in pipeline (debug only) image_size = image[0].size if isinstance(image, list) else image.size @@ -745,14 +773,18 @@ class QwenImageEditPipeline( image = self.image_processor.preprocess(image, calculated_height, calculated_width) image = image.unsqueeze(2) - num_inference_steps = req.num_inference_steps or num_inference_steps - generator = req.generator or generator - true_cfg_scale = req.true_cfg_scale or true_cfg_scale - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length + generator = req.sampling_params.generator or generator + true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) # 1. check inputs # 2. encode prompts diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py index 4e2fe35f1..52f1d64ec 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py @@ -5,7 +5,7 @@ import json import logging import os from collections.abc import Iterable -from typing import Any +from typing import Any, cast import numpy as np import PIL.Image @@ -42,6 +42,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( ) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs +from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) @@ -70,24 +71,33 @@ def get_qwen_image_edit_plus_pre_process_func( latent_channels = vae_config.get("z_dim", 16) def pre_process_func( - requests: list[OmniDiffusionRequest], + request: OmniDiffusionRequest, ): """Pre-process requests for QwenImageEditPlusPipeline.""" - for req in requests: - image = req.pil_image + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} # Handle single image or list of images - if image is None: + if raw_image is None: continue - if not isinstance(image, list): - image = [image] + if not isinstance(raw_image, list): + raw_image = [raw_image] + image = [ + PIL.Image.open(im) if isinstance(im, str) else cast(PIL.Image.Image | np.ndarray | torch.Tensor, im) + for im in raw_image + ] # Calculate dimensions based on first image image_size = image[0].size calculated_width, calculated_height = calculate_dimensions(VAE_IMAGE_SIZE, image_size[0] / image_size[1]) - height = req.height or calculated_height - width = req.width or calculated_width + height = request.sampling_params.height or calculated_height + width = request.sampling_params.width or calculated_width # Ensure dimensions are multiples of vae_scale_factor * 2 multiple_of = vae_scale_factor * 2 @@ -95,10 +105,10 @@ def get_qwen_image_edit_plus_pre_process_func( width = width // multiple_of * multiple_of # Store calculated dimensions in request - req.calculated_height = calculated_height - req.calculated_width = calculated_width - req.height = height - req.width = width + prompt["additional_information"]["calculated_height"] = calculated_height + prompt["additional_information"]["calculated_width"] = calculated_width + request.sampling_params.height = height + request.sampling_params.width = width # Preprocess images into condition_images (for prompt encoding) and vae_images (for VAE encoding) condition_images = [] @@ -124,12 +134,12 @@ def get_qwen_image_edit_plus_pre_process_func( vae_images.append(image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2)) # Store preprocessed images in request - req.condition_images = condition_images - req.vae_images = vae_images - req.condition_image_sizes = condition_image_sizes - req.vae_image_sizes = vae_image_sizes - - return requests + prompt["additional_information"]["condition_images"] = condition_images + prompt["additional_information"]["vae_images"] = vae_images + prompt["additional_information"]["condition_image_sizes"] = condition_image_sizes + prompt["additional_information"]["vae_image_sizes"] = vae_image_sizes + request.prompts[i] = prompt + return request return pre_process_func @@ -281,7 +291,7 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): def _get_qwen_prompt_embeds( self, - prompt: str | list[str] = None, + prompt: str | list[str], image: list[torch.Tensor] | torch.Tensor | None = None, dtype: torch.dtype | None = None, ): @@ -655,19 +665,31 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): max_sequence_length: int = 512, ) -> DiffusionOutput: """Forward pass for image editing with support for multiple images.""" - prompt = req.prompt - negative_prompt = req.negative_prompt + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + if len(req.prompts) > 1: + logger.warning( + """This model only supports a single prompt, not a batched request.""", + """Taking only the first image for now.""", + ) + first_prompt = req.prompts[0] + prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") + negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt") # Get preprocessed images from request (pre-processing is done in DiffusionEngine) - if hasattr(req, "vae_images") and hasattr(req, "condition_images"): - condition_images = req.condition_images - vae_images = req.vae_images - condition_image_sizes = req.condition_image_sizes - vae_image_sizes = req.vae_image_sizes - calculated_height = req.calculated_height - calculated_width = req.calculated_width - height = req.height - width = req.width + if ( + not isinstance(first_prompt, str) + and "vae_images" in (additional_information := first_prompt.get("additional_information", {})) + and "condition_images" in additional_information + ): + condition_images = additional_information.get("condition_images") + vae_images = additional_information.get("vae_images") + condition_image_sizes = additional_information.get("condition_image_sizes") + vae_image_sizes = additional_information.get("vae_image_sizes") + calculated_height = additional_information.get("calculated_height") + calculated_width = additional_information.get("calculated_width") + height = req.sampling_params.height + width = req.sampling_params.width else: # fallback to run pre-processing in pipeline (debug only) if image is None: @@ -701,14 +723,18 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): condition_images.append(self.image_processor.resize(img, condition_height, condition_width)) vae_images.append(self.image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2)) - num_inference_steps = req.num_inference_steps or num_inference_steps - generator = req.generator or generator - true_cfg_scale = req.true_cfg_scale or true_cfg_scale - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length + generator = req.sampling_params.generator or generator + true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) # 1. check inputs # 2. encode prompts diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py index ae2d039e3..ae27ba708 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py @@ -7,7 +7,7 @@ import logging import math import os from collections.abc import Iterable -from typing import Any +from typing import Any, cast import numpy as np import PIL.Image @@ -38,6 +38,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( ) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs +from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.model_executor.model_loader.weight_utils import ( download_weights_from_hf_specific, ) @@ -69,17 +70,35 @@ def get_qwen_image_layered_pre_process_func( image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) def pre_process_func( - requests: list[OmniDiffusionRequest], + request: OmniDiffusionRequest, ): """Pre-process requests for QwenImageLayeredPipeline.""" - for req in requests: - image = req.pil_image + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} + + if raw_image is None or isinstance(raw_image, list): + raise ValueError( + """Received no image or a list of image. Only a single image is supported by this model.""" + """Please correctly set `"multi_modal_data": {"image": , …}`""" + ) + + if isinstance(raw_image, str): + image = PIL.Image.open(raw_image) + else: + image = cast(PIL.Image.Image | torch.Tensor | np.ndarray, raw_image) # 1. calculate dimensions - image_size = image[0].size if isinstance(image, list) else image.size - assert req.resolution in [640, 1024], f"resolution must be either 640 or 1024, but got {req.resolution}" + image_size = image.size + assert request.sampling_params.resolution in [640, 1024], ( + f"resolution must be either 640 or 1024, but got {request.sampling_params.resolution}" + ) calculated_width, calculated_height = calculate_dimensions( - req.resolution * req.resolution, image_size[0] / image_size[1] + request.sampling_params.resolution * request.sampling_params.resolution, image_size[0] / image_size[1] ) height = calculated_height width = calculated_width @@ -89,10 +108,10 @@ def get_qwen_image_layered_pre_process_func( height = height // multiple_of * multiple_of # Store calculated dimensions in request - req.calculated_height = calculated_height - req.calculated_width = calculated_width - req.height = height - req.width = width + prompt["additional_information"]["calculated_height"] = calculated_height + prompt["additional_information"]["calculated_width"] = calculated_width + request.sampling_params.height = height + request.sampling_params.width = width # 2. Preprocess image if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == latent_channels): @@ -103,10 +122,10 @@ def get_qwen_image_layered_pre_process_func( # image = image.to(dtype=self.text_encoder.dtype) # do it later # Store preprocessed image and prompt image in request - req.preprocessed_image = image - req.prompt_image = prompt_image - - return requests + prompt["additional_information"]["preprocessed_image"] = image + prompt["additional_information"]["prompt_image"] = prompt_image + request.prompts[i] = prompt + return request return pre_process_func @@ -689,7 +708,7 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as req: OmniDiffusionRequest, image: PIL.Image.Image | torch.Tensor | None = None, prompt: str | list[str] | None = None, - negative_prompt: str | list[str] = None, + negative_prompt: str | list[str] | None = None, true_cfg_scale: float = 4.0, layers: int | None = 4, num_inference_steps: int = 50, @@ -713,29 +732,48 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as # 1. Get preprocessed image from request (pre-processing is done in DiffusionEngine) # Override parameters from request if provided - prompt = req.prompt if req.prompt is not None else prompt - layers = req.layers if req.layers is not None else layers - resolution = req.resolution if req.resolution is not None else resolution - cfg_normalize = req.cfg_normalize if req.cfg_normalize is not None else cfg_normalize - use_en_prompt = req.use_en_prompt if req.use_en_prompt is not None else use_en_prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt - num_inference_steps = req.num_inference_steps or num_inference_steps - generator = req.generator or generator - true_cfg_scale = req.true_cfg_scale or true_cfg_scale - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + if len(req.prompts) > 1: + logger.warning( + """This model only supports a single prompt, not a batched request.""", + """Taking only the first image for now.""", + ) + first_prompt = req.prompts[0] + prompt = first_prompt if isinstance(first_prompt, str) else (first_prompt.get("prompt") or "") + negative_prompt = None if isinstance(first_prompt, str) else first_prompt.get("negative_prompt") - if hasattr(req, "preprocessed_image"): - prompt_image = req.prompt_image - image = req.preprocessed_image + layers = req.sampling_params.layers if req.sampling_params.layers is not None else layers + resolution = req.sampling_params.resolution if req.sampling_params.resolution is not None else resolution + max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length + cfg_normalize = ( + req.sampling_params.cfg_normalize if req.sampling_params.cfg_normalize is not None else cfg_normalize + ) + use_en_prompt = ( + req.sampling_params.use_en_prompt if req.sampling_params.use_en_prompt is not None else use_en_prompt + ) + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + generator = req.sampling_params.generator or generator + true_cfg_scale = req.sampling_params.true_cfg_scale or true_cfg_scale + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) + + if not isinstance(first_prompt, str) and "preprocessed_image" in ( + additional_information := first_prompt.get("additional_information", {}) + ): + prompt_image = additional_information.get("prompt_image") + image = additional_information.get("preprocessed_image") image = image.to(dtype=self.text_encoder.dtype) # Now we get the type - calculated_height = req.calculated_height - calculated_width = req.calculated_width - height = req.height - width = req.width + calculated_height = additional_information.get("calculated_height") + calculated_width = additional_information.get("calculated_width") + height = req.sampling_params.height + width = req.sampling_params.width else: # fallback to run pre-processing in pipeline (debug only) image_size = image[0].size if isinstance(image, list) else image.size diff --git a/vllm_omni/diffusion/models/sd3/pipeline_sd3.py b/vllm_omni/diffusion/models/sd3/pipeline_sd3.py index dd17af883..34a0eb6c1 100644 --- a/vllm_omni/diffusion/models/sd3/pipeline_sd3.py +++ b/vllm_omni/diffusion/models/sd3/pipeline_sd3.py @@ -562,19 +562,24 @@ class StableDiffusion3Pipeline( negative_pooled_prompt_embeds: torch.Tensor | None = None, max_sequence_length: int = 256, ) -> DiffusionOutput: - # # TODO: only support single prompt now - # if req.prompt is not None: - # prompt = req.prompt[0] if isinstance(req.prompt, list) else req.prompt - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt - height = req.height or self.default_sample_size * self.vae_scale_factor - width = req.width or self.default_sample_size * self.vae_scale_factor - sigmas = req.sigmas or sigmas - num_inference_steps = req.num_inference_steps or num_inference_steps - generator = req.generator or generator - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt + negative_prompt = [ + "" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts + ] or negative_prompt + + height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor + width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor + sigmas = req.sampling_params.sigmas or sigmas + max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + generator = req.sampling_params.generator or generator + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) # 1. check inputs # 2. encode prompts # 3. prepare latents and timesteps @@ -595,7 +600,7 @@ class StableDiffusion3Pipeline( max_sequence_length=max_sequence_length, ) - self._guidance_scale = req.guidance_scale + self._guidance_scale = req.sampling_params.guidance_scale self._current_timestep = None self._interrupt = False diff --git a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py index c947bd934..c48d68efd 100644 --- a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py +++ b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py @@ -332,7 +332,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput): sample_size: int, dtype: torch.dtype, device: torch.device, - generator: torch.Generator | None, + generator: torch.Generator | list[torch.Generator] | None, latents: torch.Tensor | None = None, ) -> torch.Tensor: """Prepare initial latent noise.""" @@ -357,7 +357,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput): num_inference_steps: int = 100, guidance_scale: float = 7.0, num_waveforms_per_prompt: int = 1, - generator: torch.Generator | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, latents: torch.Tensor | None = None, prompt_embeds: torch.Tensor | None = None, negative_prompt_embeds: torch.Tensor | None = None, @@ -385,20 +385,26 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput): DiffusionOutput containing generated audio """ # Extract from request - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt - num_inference_steps = req.num_inference_steps or num_inference_steps - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt + if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): + negative_prompt = None + elif req.prompts: + negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] + + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale if generator is None: - generator = req.generator - if generator is None and req.seed is not None: - generator = torch.Generator(device=self.device).manual_seed(req.seed) + generator = req.sampling_params.generator + if generator is None and req.sampling_params.seed is not None: + generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed) # Get audio duration from request extra params or defaults - audio_start_in_s = req.extra.get("audio_start_in_s", audio_start_in_s) - audio_end_in_s = req.extra.get("audio_end_in_s", audio_end_in_s) + audio_start_in_s = req.sampling_params.extra_args.get("audio_start_in_s", audio_start_in_s) + audio_end_in_s = req.sampling_params.extra_args.get("audio_end_in_s", audio_end_in_s) # Calculate audio length downsample_ratio = self.vae.hop_length diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index b691cfedb..712c05400 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -4,8 +4,10 @@ from __future__ import annotations import json +import logging import os from collections.abc import Iterable +from typing import cast import PIL.Image import torch @@ -21,6 +23,9 @@ from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineL from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniTextPrompt + +logger = logging.getLogger(__name__) def retrieve_latents( @@ -127,39 +132,54 @@ def get_wan22_pre_process_func( video_processor = VideoProcessor(vae_scale_factor=8) - def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]: - for req in requests: - # Load image if path is provided - if req.image_path is not None and req.pil_image is None: - req.pil_image = PIL.Image.open(req.image_path).convert("RGB") + def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest: + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} - if req.pil_image is not None: - image = req.pil_image + if raw_image is None: + continue - # Calculate dimensions based on aspect ratio if not provided - if req.height is None or req.width is None: - # Default max area for 720P - max_area = 720 * 1280 - aspect_ratio = image.height / image.width + if not isinstance(raw_image, (str, PIL.Image.Image)): + raise TypeError( + f"""Unsupported image format {raw_image.__class__}.""", + """Please correctly set `"multi_modal_data": {"image": , …}`""", + ) + image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image - # Calculate dimensions maintaining aspect ratio - mod_value = 16 # Must be divisible by 16 - height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value - width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value + # Calculate dimensions based on aspect ratio if not provided + if request.sampling_params.height is None or request.sampling_params.width is None: + # Default max area for 720P + max_area = 720 * 1280 + aspect_ratio = image.height / image.width - if req.height is None: - req.height = height - if req.width is None: - req.width = width + # Calculate dimensions maintaining aspect ratio + mod_value = 16 # Must be divisible by 16 + height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value + width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value - # Resize image to target dimensions - image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS) - req.pil_image = image + if request.sampling_params.height is None: + request.sampling_params.height = height + if request.sampling_params.width is None: + request.sampling_params.width = width - # Preprocess for VAE - req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width) + # Resize image to target dimensions + image = image.resize( + (request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None + PIL.Image.Resampling.LANCZOS, + ) + prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above - return requests + # Preprocess for VAE + prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess( + image, height=request.sampling_params.height, width=request.sampling_params.width + ) + request.prompts[i] = prompt + return request return pre_process_func @@ -289,20 +309,27 @@ class Wan22Pipeline(nn.Module): guidance_scale: float | tuple[float, float] = 4.0, frame_num: int = 81, output_type: str | None = "np", - generator: torch.Generator | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, prompt_embeds: torch.Tensor | None = None, negative_prompt_embeds: torch.Tensor | None = None, attention_kwargs: dict | None = None, **kwargs, ) -> DiffusionOutput: - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt + # Get parameters from request or arguments + if len(req.prompts) > 1: + raise ValueError( + """This model only supports a single prompt, not a batched request.""", + """Please pass in a single prompt object or string, or a single-item list.""", + ) + if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list + prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt") + negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt") if prompt is None and prompt_embeds is None: raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.") - height = req.height or height - width = req.width or width - num_frames = req.num_frames if req.num_frames else frame_num + height = req.sampling_params.height or height + width = req.sampling_params.width or width + num_frames = req.sampling_params.num_frames if req.sampling_params.num_frames else frame_num # Ensure dimensions are compatible with VAE and patch size # For expand_timesteps mode, we need latent dims to be even (divisible by patch_size) @@ -310,16 +337,16 @@ class Wan22Pipeline(nn.Module): mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V height = (height // mod_value) * mod_value width = (width // mod_value) * mod_value - num_steps = req.num_inference_steps or num_inference_steps + num_steps = req.sampling_params.num_inference_steps or num_inference_steps # Respect per-request guidance_scale when explicitly provided. - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0] guidance_high = ( - req.guidance_scale_2 - if req.guidance_scale_2 is not None + req.sampling_params.guidance_scale_2 + if req.sampling_params.guidance_scale_2 is not None else ( guidance_scale[1] if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1 @@ -351,9 +378,9 @@ class Wan22Pipeline(nn.Module): # Seed / generator if generator is None: - generator = req.generator - if generator is None and req.seed is not None: - generator = torch.Generator(device=device).manual_seed(req.seed) + generator = req.sampling_params.generator + if generator is None and req.sampling_params.seed is not None: + generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) # Encode prompts if prompt_embeds is None: @@ -361,8 +388,8 @@ class Wan22Pipeline(nn.Module): prompt=prompt, negative_prompt=negative_prompt, do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0, - num_videos_per_prompt=req.num_outputs_per_prompt or 1, - max_sequence_length=req.max_sequence_length or 512, + num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1, + max_sequence_length=req.sampling_params.max_sequence_length or 512, device=device, dtype=dtype, ) @@ -384,7 +411,22 @@ class Wan22Pipeline(nn.Module): boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps # Handle I2V mode when expand_timesteps=True and image is provided - image = req.pil_image + multi_modal_data = req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(raw_image, list): + if len(raw_image) > 1: + logger.warning( + """Received a list of image. Only a single image is supported by this model.""" + """Taking only the first image for now.""" + ) + raw_image = raw_image[0] + if raw_image is None: + image = None + elif isinstance(raw_image, str): + image = PIL.Image.open(raw_image) + else: + image = cast(PIL.Image.Image | torch.Tensor, raw_image) + latent_condition = None first_frame_mask = None @@ -415,7 +457,7 @@ class Wan22Pipeline(nn.Module): dtype=torch.float32, device=device, generator=generator, - latents=req.latents, + latents=req.sampling_params.latents, ) # Encode image condition @@ -457,7 +499,7 @@ class Wan22Pipeline(nn.Module): dtype=torch.float32, device=device, generator=generator, - latents=req.latents, + latents=req.sampling_params.latents, ) if attention_kwargs is None: diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index ed8c0a987..ea045e5d6 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -3,8 +3,10 @@ from __future__ import annotations +import logging import os from collections.abc import Iterable +from typing import cast import numpy as np import PIL.Image @@ -26,6 +28,9 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( retrieve_latents, ) from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniTextPrompt + +logger = logging.getLogger(__name__) def _load_model_index(model: str, local_files_only: bool) -> dict: @@ -77,39 +82,56 @@ def get_wan22_i2v_pre_process_func( video_processor = VideoProcessor(vae_scale_factor=8) - def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]: - for req in requests: - # Load image if path is provided - if req.image_path is not None and req.pil_image is None: - req.pil_image = PIL.Image.open(req.image_path).convert("RGB") + def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest: + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} - if req.pil_image is not None: - image = req.pil_image + if raw_image is None: + raise ValueError( + """No image is provided. This model requires an image to run.""", + """Please correctly set `"multi_modal_data": {"image": , …}`""", + ) + if not isinstance(raw_image, (str, PIL.Image.Image)): + raise TypeError( + f"""Unsupported image format {raw_image.__class__}.""", + """Please correctly set `"multi_modal_data": {"image": , …}`""", + ) + image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image - # Calculate dimensions based on aspect ratio if not provided - if req.height is None or req.width is None: - # Default max area for 480P - max_area = 480 * 832 - aspect_ratio = image.height / image.width + # Calculate dimensions based on aspect ratio if not provided + if request.sampling_params.height is None or request.sampling_params.width is None: + # Default max area for 480P + max_area = 480 * 832 + aspect_ratio = image.height / image.width - # Calculate dimensions maintaining aspect ratio - mod_value = 16 # Must be divisible by 16 - height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value - width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value + # Calculate dimensions maintaining aspect ratio + mod_value = 16 # Must be divisible by 16 + height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value + width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value - if req.height is None: - req.height = height - if req.width is None: - req.width = width + if request.sampling_params.height is None: + request.sampling_params.height = height + if request.sampling_params.width is None: + request.sampling_params.width = width - # Resize image to target dimensions - image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS) - req.pil_image = image + # Resize image to target dimensions + image = image.resize( + (request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None + PIL.Image.Resampling.LANCZOS, + ) + prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above - # Preprocess for VAE - req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width) - - return requests + # Preprocess for VAE + prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess( + image, height=request.sampling_params.height, width=request.sampling_params.width + ) + request.prompts[i] = prompt + return request return pre_process_func @@ -266,7 +288,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): guidance_scale: float | tuple[float, float] = 5.0, frame_num: int = 81, output_type: str | None = "np", - generator: torch.Generator | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, prompt_embeds: torch.Tensor | None = None, negative_prompt_embeds: torch.Tensor | None = None, image_embeds: torch.Tensor | None = None, @@ -275,31 +297,51 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): **kwargs, ) -> DiffusionOutput: # Get parameters from request or arguments - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt + if len(req.prompts) > 1: + raise ValueError( + """This model only supports a single prompt, not a batched request.""", + """Please pass in a single prompt object or string, or a single-item list.""", + ) + if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list + prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt") + negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt") if prompt is None and prompt_embeds is None: - raise ValueError("Prompt or prompt_embeds is required for Wan2.2 I2V generation.") + raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.") # Get image from request if image is None: - image = req.pil_image - if image is None: - raise ValueError("Image is required for I2V generation.") + multi_modal_data = ( + req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None + ) + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if raw_image is None: + raise ValueError("Image is required for I2V generation.") + if isinstance(raw_image, list): + if len(raw_image) > 1: + logger.warning( + """Received a list of image. Only a single image is supported by this model.""" + """Taking only the first image for now.""" + ) + raw_image = raw_image[0] + if isinstance(raw_image, str): + image = PIL.Image.open(raw_image) + else: + image = cast(PIL.Image.Image | torch.Tensor, raw_image) - height = req.height or height - width = req.width or width - num_frames = req.num_frames if req.num_frames else frame_num - num_steps = req.num_inference_steps or num_inference_steps + height = req.sampling_params.height or height + width = req.sampling_params.width or width + num_frames = req.sampling_params.num_frames or frame_num + num_steps = req.sampling_params.num_inference_steps or num_inference_steps # Respect per-request guidance_scale when explicitly provided. - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale # Handle guidance scales guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0] guidance_high = ( - req.guidance_scale_2 - if req.guidance_scale_2 is not None + req.sampling_params.guidance_scale_2 + if req.sampling_params.guidance_scale_2 is not None else ( guidance_scale[1] if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1 @@ -333,9 +375,9 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): # Generator setup if generator is None: - generator = req.generator - if generator is None and req.seed is not None: - generator = torch.Generator(device=device).manual_seed(req.seed) + generator = req.sampling_params.generator + if generator is None and req.sampling_params.seed is not None: + generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) # Encode prompts if prompt_embeds is None: @@ -343,8 +385,8 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): prompt=prompt, negative_prompt=negative_prompt, do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0, - num_videos_per_prompt=req.num_outputs_per_prompt or 1, - max_sequence_length=req.max_sequence_length or 512, + num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1, + max_sequence_length=req.sampling_params.max_sequence_length or 512, device=device, dtype=dtype, ) @@ -410,7 +452,7 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): dtype=torch.float32, device=device, generator=generator, - latents=req.latents, + latents=req.sampling_params.latents, last_image=last_image_tensor, ) diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py index b943000cf..5944b678a 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py @@ -16,8 +16,10 @@ The key difference from the MoE-based I2V pipeline is: from __future__ import annotations +import logging import os from collections.abc import Iterable +from typing import cast import numpy as np import PIL.Image @@ -39,6 +41,9 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( retrieve_latents, ) from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniTextPrompt + +logger = logging.getLogger(__name__) def get_wan22_ti2v_post_process_func( @@ -67,39 +72,56 @@ def get_wan22_ti2v_pre_process_func( video_processor = VideoProcessor(vae_scale_factor=8) - def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]: - for req in requests: - # Load image if path is provided - if req.image_path is not None and req.pil_image is None: - req.pil_image = PIL.Image.open(req.image_path).convert("RGB") + def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest: + for i, prompt in enumerate(request.prompts): + multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(prompt, str): + prompt = OmniTextPrompt(prompt=prompt) + if "additional_information" not in prompt: + prompt["additional_information"] = {} - if req.pil_image is not None: - image = req.pil_image + if raw_image is None: + raise ValueError( + """No image is provided. This model requires an image to run.""", + """Please correctly set `"multi_modal_data": {"image": , …}`""", + ) + if not isinstance(raw_image, (str, PIL.Image.Image)): + raise TypeError( + f"""Unsupported image format {raw_image.__class__}.""", + """Please correctly set `"multi_modal_data": {"image": , …}`""", + ) + image = PIL.Image.open(raw_image).convert("RGB") if isinstance(raw_image, str) else raw_image - # Calculate dimensions based on aspect ratio if not provided - if req.height is None or req.width is None: - # Default max area for 720P (TI2V-5B default) - max_area = 720 * 1280 - aspect_ratio = image.height / image.width + # Calculate dimensions based on aspect ratio if not provided + if request.sampling_params.height is None or request.sampling_params.width is None: + # Default max area for 720P (TI2V-5B default) + max_area = 720 * 1280 + aspect_ratio = image.height / image.width - # Calculate dimensions maintaining aspect ratio - mod_value = 16 # Must be divisible by 16 - height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value - width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value + # Calculate dimensions maintaining aspect ratio + mod_value = 16 # Must be divisible by 16 + height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value + width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value - if req.height is None: - req.height = height - if req.width is None: - req.width = width + if request.sampling_params.height is None: + request.sampling_params.height = height + if request.sampling_params.width is None: + request.sampling_params.width = width - # Resize image to target dimensions - image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS) - req.pil_image = image + # Resize image to target dimensions + image = image.resize( + (request.sampling_params.width, request.sampling_params.height), # type: ignore # Above has ensured that width & height are not None + PIL.Image.Resampling.LANCZOS, + ) + prompt["multi_modal_data"]["image"] = image # type: ignore # key existence already checked above - # Preprocess for VAE - req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width) - - return requests + # Preprocess for VAE + prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess( + image, height=request.sampling_params.height, width=request.sampling_params.width + ) + request.prompts[i] = prompt + return request return pre_process_func @@ -205,31 +227,53 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): guidance_scale: float = 5.0, frame_num: int = 81, output_type: str | None = "np", - generator: torch.Generator | None = None, + generator: torch.Generator | list[torch.Generator] | None = None, prompt_embeds: torch.Tensor | None = None, negative_prompt_embeds: torch.Tensor | None = None, attention_kwargs: dict | None = None, **kwargs, ) -> DiffusionOutput: # Get parameters from request or arguments - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt + if len(req.prompts) > 1: + raise ValueError( + """This model only supports a single prompt, not a batched request.""", + """Please pass in a single prompt object or string, or a single-item list.""", + ) + if len(req.prompts) == 1: # If req.prompt is empty, default to prompt & neg_prompt in param list + prompt = req.prompts[0] if isinstance(req.prompts[0], str) else req.prompts[0].get("prompt") + negative_prompt = None if isinstance(req.prompts[0], str) else req.prompts[0].get("negative_prompt") if prompt is None and prompt_embeds is None: - raise ValueError("Prompt or prompt_embeds is required for Wan2.2 TI2V generation.") + raise ValueError("Prompt or prompt_embeds is required for Wan2.2 generation.") # Get image from request (optional for TI2V) if image is None: - image = req.pil_image + multi_modal_data = ( + req.prompts[0].get("multi_modal_data", {}) if not isinstance(req.prompts[0], str) else None + ) + raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None + if isinstance(raw_image, list): + if len(raw_image) > 1: + logger.warning( + """Received a list of image. Only a single image is supported by this model.""" + """Taking only the first image for now.""" + ) + raw_image = raw_image[0] + if raw_image is None: + image = None + elif isinstance(raw_image, str): + image = PIL.Image.open(raw_image) + else: + image = cast(PIL.Image.Image | torch.Tensor, raw_image) # Default dimensions for TI2V-5B (720P) - height = req.height or height - width = req.width or width - num_frames = req.num_frames if req.num_frames else frame_num - num_steps = req.num_inference_steps or num_inference_steps + height = req.sampling_params.height or height + width = req.sampling_params.width or width + num_frames = req.sampling_params.num_frames if req.sampling_params.num_frames else frame_num + num_steps = req.sampling_params.num_inference_steps or num_inference_steps # Respect per-request guidance_scale when explicitly provided. - if req.guidance_scale_provided: - guidance_scale = req.guidance_scale + if req.sampling_params.guidance_scale_provided: + guidance_scale = req.sampling_params.guidance_scale self._guidance_scale = guidance_scale @@ -254,9 +298,9 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): # Generator setup if generator is None: - generator = req.generator - if generator is None and req.seed is not None: - generator = torch.Generator(device=device).manual_seed(req.seed) + generator = req.sampling_params.generator + if generator is None and req.sampling_params.seed is not None: + generator = torch.Generator(device=device).manual_seed(req.sampling_params.seed) # Encode prompts if prompt_embeds is None: @@ -264,8 +308,8 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): prompt=prompt, negative_prompt=negative_prompt, do_classifier_free_guidance=guidance_scale > 1.0, - num_videos_per_prompt=req.num_outputs_per_prompt or 1, - max_sequence_length=req.max_sequence_length or 512, + num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1, + max_sequence_length=req.sampling_params.max_sequence_length or 512, device=device, dtype=dtype, ) @@ -310,7 +354,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): dtype=torch.float32, device=device, generator=generator, - latents=req.latents, + latents=req.sampling_params.latents, ) else: # T2V mode: prepare random latents @@ -323,7 +367,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): dtype=torch.float32, device=device, generator=generator, - latents=req.latents, + latents=req.sampling_params.latents, ) latent_condition = None first_frame_mask = torch.ones( diff --git a/vllm_omni/diffusion/models/z_image/pipeline_z_image.py b/vllm_omni/diffusion/models/z_image/pipeline_z_image.py index b92d3de88..62d92900a 100644 --- a/vllm_omni/diffusion/models/z_image/pipeline_z_image.py +++ b/vllm_omni/diffusion/models/z_image/pipeline_z_image.py @@ -316,15 +316,15 @@ class ZImagePipeline(nn.Module): self, req: OmniDiffusionRequest, prompt: str | list[str] | None = None, - height: int | None = None, - width: int | None = None, + height: int = 1024, + width: int = 1024, num_inference_steps: int = 50, sigmas: list[float] | None = None, guidance_scale: float = 5.0, cfg_normalization: bool = False, cfg_truncation: float = 1.0, negative_prompt: str | list[str] | None = None, - num_images_per_prompt: int | None = 1, + num_images_per_prompt: int = 1, generator: torch.Generator | list[torch.Generator] | None = None, latents: torch.FloatTensor | None = None, prompt_embeds: list[torch.FloatTensor] | None = None, @@ -413,16 +413,28 @@ class ZImagePipeline(nn.Module): `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated images. """ - prompt = req.prompt - negative_prompt = req.negative_prompt - height: int = req.height or 1024 - width: int = req.width or 1024 - num_inference_steps = req.num_inference_steps or 50 - generator = req.generator - guidance_scale = req.guidance_scale if req.guidance_rescale is not None else guidance_scale - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt + if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): + negative_prompt = None + elif req.prompts: + negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] + + height = req.sampling_params.height or height + width = req.sampling_params.width or width + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + generator = req.sampling_params.generator + sigmas = req.sampling_params.sigmas or sigmas + max_sequence_length = req.sampling_params.max_sequence_length or max_sequence_length + guidance_scale = ( + req.sampling_params.guidance_scale if req.sampling_params.guidance_rescale is not None else guidance_scale + ) + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) vae_scale = self.vae_scale_factor * 2 if height % vae_scale != 0: diff --git a/vllm_omni/diffusion/request.py b/vllm_omni/diffusion/request.py index 89c0a79f1..a6005290c 100644 --- a/vllm_omni/diffusion/request.py +++ b/vllm_omni/diffusion/request.py @@ -2,14 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import pprint -from dataclasses import asdict, dataclass, field -from typing import Any +from dataclasses import dataclass, field -import PIL.Image -import torch - -from vllm_omni.lora.request import LoRARequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType @dataclass @@ -17,9 +12,8 @@ class OmniDiffusionRequest: """ Complete state passed through the pipeline execution. - This dataclass contains all information needed during the diffusion pipeline - execution, allowing methods to update specific components without needing - to manage numerous individual parameters. + This dataclass contains the prompts and sampling parameters for the diffusion pipeline + execution. It also contains a request_id for other components to trace this request and its outputs. """ # TODO(will): double check that args are separate from server_args @@ -27,173 +21,24 @@ class OmniDiffusionRequest: # specific arguments. # data_type: DataType - request_id: str | None = None + prompts: list[OmniPromptType] # Actually supporting str-based prompts + sampling_params: OmniDiffusionSamplingParams - generator: torch.Generator | list[torch.Generator] | None = None - - # Image inputs - image_path: str | None = None - # Image encoder hidden states - image_embeds: list[torch.Tensor] = field(default_factory=list) - pil_image: torch.Tensor | PIL.Image.Image | None = None - pixel_values: torch.Tensor | PIL.Image.Image | None = None - preprocessed_image: torch.Tensor | None = None - - # Text inputs - prompt: str | list[str] | None = None - negative_prompt: str | list[str] | None = None - prompt_path: str | None = None - output_path: str = "outputs/" - # without extension - output_file_name: str | None = None - output_file_ext: str | None = None - # Primary encoder embeddings - prompt_embeds: list[torch.Tensor] | torch.Tensor = field(default_factory=list) - negative_prompt_embeds: list[torch.Tensor] | None = None - prompt_attention_mask: list[torch.Tensor] | None = None - negative_attention_mask: list[torch.Tensor] | None = None - clip_embedding_pos: list[torch.Tensor] | None = None - clip_embedding_neg: list[torch.Tensor] | None = None - - pooled_embeds: list[torch.Tensor] = field(default_factory=list) - neg_pooled_embeds: list[torch.Tensor] = field(default_factory=list) - - # Additional text-related parameters - max_sequence_length: int | None = None - prompt_template: dict[str, Any] | None = None - do_classifier_free_guidance: bool = False - - # Batch info - num_outputs_per_prompt: int = 1 - seed: int | None = None - seeds: list[int] | None = None - - # layered info - layers: int = 4 - - # cfg info - cfg_normalize: bool = False - - # caption language - use_en_prompt: bool = False - - # different bucket in (640, 1024) to determine the condition and output resolution - resolution: int = 640 - - # Tracking if embeddings are already processed - is_prompt_processed: bool = False - - # Latent tensors - latents: torch.Tensor | None = None - raw_latent_shape: torch.Tensor | None = None - noise_pred: torch.Tensor | None = None - image_latent: torch.Tensor | None = None - - # Latent dimensions - height_latents: list[int] | int | None = None - width_latents: list[int] | int | None = None - num_frames: list[int] | int = 1 # Default for image models - num_frames_round_down: bool = False # Whether to round down num_frames if it's not divisible by num_gpus - - # Original dimensions (before VAE scaling) - height: list[int] | int | None = None - width: list[int] | int | None = None - fps: list[int] | int | None = None - height_not_provided: bool = False - width_not_provided: bool = False - - # Timesteps - timesteps: torch.Tensor | None = None - timestep: torch.Tensor | float | int | None = None - step_index: int | None = None - boundary_ratio: float | None = None - - # Scheduler parameters - num_inference_steps: int = 50 - guidance_scale: float = 1.0 - guidance_scale_provided: bool = False - guidance_scale_2: float | None = None - guidance_rescale: float = 0.0 - eta: float = 0.0 - sigmas: list[float] | None = None - - true_cfg_scale: float | None = None # qwen-image specific now - - n_tokens: int | None = None - - # Other parameters that may be needed by specific schedulers - extra_step_kwargs: dict[str, Any] = field(default_factory=dict) - - # [Omni] KV Cache Transfer, for bagel model now - past_key_values: Any | None = None # Injected KV Cache - kv_metadata: dict[str, Any] | None = None # Metadata for KV Cache (e.g., kv_lens, ropes) - need_kv_receive: bool = True # Flag to indicate if this request expects KV transfer - - # Component modules (populated by the pipeline) - modules: dict[str, Any] = field(default_factory=dict) - - return_trajectory_latents: bool = False - return_trajectory_decoded: bool = False - trajectory_timesteps: list[torch.Tensor] | None = None - trajectory_latents: torch.Tensor | None = None - - # Extra parameters that might be needed by specific pipeline implementations - extra: dict[str, Any] = field(default_factory=dict) - - # Misc - save_output: bool = True - return_frames: bool = False - - # LoRA - lora_request: LoRARequest | None = None - lora_scale: float = 1.0 - - # STA parameters - STA_param: list | None = None - is_cfg_negative: bool = False - mask_search_final_result_pos: list[list] | None = None - mask_search_final_result_neg: list[list] | None = None - - # VSA parameters - VSA_sparsity: float = 0.0 - # perf_logger: PerformanceLogger | None = None - - # stage logging - # logging_info: PipelineLoggingInfo = field(default_factory=PipelineLoggingInfo) - - # profile - profile: bool = False - num_profiled_timesteps: int = 8 - - # debugging - debug: bool = False - - # results - output: torch.Tensor | None = None - - @property - def batch_size(self): - # Determine batch size - if isinstance(self.prompt, list): - batch_size = len(self.prompt) - elif self.prompt is not None: - batch_size = 1 - else: - batch_size = self.prompt_embeds[0].shape[0] - - # Adjust batch size for number of videos per prompt - batch_size *= self.num_outputs_per_prompt - return batch_size + request_ids: list[str] = field(default_factory=list) def __post_init__(self): """Initialize dependent fields after dataclass initialization.""" # Set do_classifier_free_guidance based on guidance scale and negative prompt - if self.guidance_scale > 1.0 and self.negative_prompt is not None: - self.do_classifier_free_guidance = True - if self.negative_prompt_embeds is None: - self.negative_prompt_embeds = [] - if self.guidance_scale_2 is None: - self.guidance_scale_2 = self.guidance_scale + if self.sampling_params.guidance_scale > 1.0 and any( + (not isinstance(p, str) and p.get("negative_prompt")) for p in self.prompts + ): + self.sampling_params.do_classifier_free_guidance = True + if self.sampling_params.guidance_scale_2 is None: + self.sampling_params.guidance_scale_2 = self.sampling_params.guidance_scale - def __str__(self): - return pprint.pformat(asdict(self), indent=2, width=120) + # The dataclass default value is 0 (false-like), used to detect whether user explicitly provides this value + # After this check is done, reset this value to old default 1 + if self.sampling_params.guidance_scale: + self.sampling_params.guidance_scale_provided = True + else: + self.sampling_params.guidance_scale = 1.0 diff --git a/vllm_omni/diffusion/scheduler.py b/vllm_omni/diffusion/scheduler.py index 21b0d4914..f104d0522 100644 --- a/vllm_omni/diffusion/scheduler.py +++ b/vllm_omni/diffusion/scheduler.py @@ -41,14 +41,14 @@ class Scheduler: def get_broadcast_handle(self): return self.mq.export_handle() - def add_req(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput: + def add_req(self, request: OmniDiffusionRequest) -> DiffusionOutput: """Sends a request to the scheduler and waits for the response.""" try: # Prepare RPC request for generation rpc_request = { "type": "rpc", "method": "generate", - "args": (requests,), + "args": (request,), "kwargs": {}, "output_rank": 0, "exec_all_ranks": True, diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py b/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py index 54327be81..bf56df155 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py @@ -173,12 +173,13 @@ class GPUDiffusionModelRunner: def _receive_kv_cache_for_request(self, req: OmniDiffusionRequest) -> None: """Receive KV cache for a request via OmniConnector.""" # TODO(wzliu)! must get control info from stage queue instead of hardcode - if not req.request_id: + if not req.request_ids: logger.warning("Request has no ID, cannot receive KV cache") return + request_id = req.request_ids[0] try: - logger.info(f"Attempting to receive KV cache for request {req.request_id}") + logger.info(f"Attempting to receive KV cache for request {request_id}") # TODO: Key used for transfer (must match sender side) # key = f"kv_cache_{req.request_id}" @@ -197,7 +198,7 @@ class GPUDiffusionModelRunner: from_stage = stage_id - 1 else: raise ValueError("Invalid stage id") - logger.info(f"Wait for KV cache for request {req.request_id} from stage {from_stage} to {to_stage}...") + logger.info(f"Wait for KV cache for request {request_id} from stage {from_stage} to {to_stage}...") # Check if we should receive KV cache based on config need_recv_cache = omni_kv_config.get("need_recv_cache", False) @@ -207,7 +208,7 @@ class GPUDiffusionModelRunner: start_time = time.time() while True: - get_key = f"omni_{from_stage}_to_{to_stage}_kv_cache_{req.request_id}" + get_key = f"omni_{from_stage}_to_{to_stage}_kv_cache_{request_id}" result = self.connector.get( from_stage=from_stage, to_stage=to_stage, @@ -217,18 +218,18 @@ class GPUDiffusionModelRunner: break if time.time() - start_time > timeout: - logger.error(f"Timeout waiting for KV cache for request {req.request_id} after {timeout}s") + logger.error(f"Timeout waiting for KV cache for request {request_id} after {timeout}s") result = None break time.sleep(0.5) else: - logger.info(f"Skip receiving KV cache for {req.request_id} (need_recv_cache=False)") + logger.info(f"Skip receiving KV cache for {request_id} (need_recv_cache=False)") result = None if result: data, size = result - logger.info(f"Successfully received KV cache for {req.request_id}") + logger.info(f"Successfully received KV cache for {request_id}") # Assume data structure matches KVCacheTransferData.to_dict() if isinstance(data, dict) and "layer_blocks" in data: @@ -242,16 +243,16 @@ class GPUDiffusionModelRunner: cache_list[i] = tensor.to(self.pipeline.device).contiguous() from types import SimpleNamespace - req.past_key_values = SimpleNamespace(**layer_blocks) + req.sampling_params.past_key_values = SimpleNamespace(**layer_blocks) if "metadata" in data: - req.kv_metadata = data["metadata"] + req.sampling_params.kv_metadata = data["metadata"] else: - logger.warning(f"No KV cache received for {req.request_id} (timeout or empty)") + logger.warning(f"No KV cache received for {request_id} (timeout or empty)") except Exception as e: - logger.error(f"Error receiving KV cache for {req.request_id}: {e}") + logger.error(f"Error receiving KV cache for {request_id}: {e}") import traceback traceback.print_exc() @@ -261,33 +262,30 @@ class GPUDiffusionModelRunner: return self.pipeline.load_weights(weights) @torch.inference_mode() - def execute_model(self, reqs: list[OmniDiffusionRequest]) -> DiffusionOutput: + def execute_model(self, req: OmniDiffusionRequest) -> DiffusionOutput: """ Execute a forward pass for the given requests. Args: - reqs: List of diffusion requests to process. + req: A diffusion request containing a list of prompts to process. Returns: DiffusionOutput with generated results. """ assert self.pipeline is not None, "Model not loaded. Call load_model() first." - if not reqs or len(reqs) == 0: + if len(req.prompts) == 0: raise ValueError("Cannot execute model with empty request list") - # TODO: dealing with first req for now - req = reqs[0] - # [Omni] KV Cache Receiving Logic - if getattr(req, "need_kv_receive", False) and self.connector is not None: + if req.sampling_params.need_kv_receive and self.connector is not None: self._receive_kv_cache_for_request(req) - if req.generator is None and req.seed is not None: - req.generator = torch.Generator(device=self.device).manual_seed(req.seed) + if req.sampling_params.generator is None and req.sampling_params.seed is not None: + req.sampling_params.generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed) # Refresh cache context if needed if self.cache_backend is not None and self.cache_backend.is_enabled(): - self.cache_backend.refresh(self.pipeline, req.num_inference_steps) + self.cache_backend.refresh(self.pipeline, req.sampling_params.num_inference_steps) with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config): with record_function("pipeline_forward"): diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py b/vllm_omni/diffusion/worker/gpu_diffusion_worker.py index cadb6b64d..d8b962a58 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py +++ b/vllm_omni/diffusion/worker/gpu_diffusion_worker.py @@ -124,9 +124,9 @@ class GPUDiffusionWorker: ) logger.info(f"Worker {self.rank}: Initialization complete.") - def generate(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput: + def generate(self, request: OmniDiffusionRequest) -> DiffusionOutput: """Generate output for the given requests.""" - return self.execute_model(requests, self.od_config) + return self.execute_model(request, self.od_config) @classmethod def start_profile(cls, trace_path_template: str) -> str: @@ -138,37 +138,17 @@ class GPUDiffusionWorker: """Stop profiling and return the result dictionary.""" return CurrentProfiler.stop() - def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput: + def execute_model(self, req: OmniDiffusionRequest, od_config: OmniDiffusionConfig) -> DiffusionOutput: """Execute a forward pass by delegating to the model runner.""" assert self.model_runner is not None, "Model runner not initialized" - if self.lora_manager is not None and reqs: - req = reqs[0] - - if len(reqs) > 1: - # This worker (and the current diffusion model runner) applies - # a single LoRA to the whole batch. Reject inconsistent LoRA - # settings to avoid silently applying the wrong adapter. - def _lora_key(r: OmniDiffusionRequest): - if r.lora_request is None: - return None - lr = r.lora_request - return (lr.lora_name, lr.lora_int_id, lr.lora_path, lr.tensorizer_config_dict) - - key0 = _lora_key(req) - scale0 = req.lora_scale if key0 is not None else None - for other in reqs[1:]: - if _lora_key(other) != key0: - raise ValueError("All requests in a diffusion batch must share the same LoRARequest.") - if key0 is not None and other.lora_scale != scale0: - raise ValueError("All requests in a diffusion batch must share the same lora_scale.") - + if self.lora_manager is not None: try: - self.lora_manager.set_active_adapter(req.lora_request, req.lora_scale) + self.lora_manager.set_active_adapter(req.sampling_params.lora_request, req.sampling_params.lora_scale) except Exception as exc: - if req.lora_request is not None: + if req.sampling_params.lora_request is not None: raise logger.warning("LoRA activation skipped: %s", exc) - return self.model_runner.execute_model(reqs) + return self.model_runner.execute_model(req) def load_weights(self, weights) -> set[str]: """Load weights by delegating to the model runner.""" @@ -360,7 +340,7 @@ class WorkerProc: except Exception as e: logger.error(f"Error processing RPC: {e}", exc_info=True) if self.result_mq is not None: - self.return_result({"status": "error", "error": str(e)}) + self.return_result(DiffusionOutput(error=str(e))) elif isinstance(msg, dict) and msg.get("type") == "shutdown": logger.info("Worker %s: Received shutdown message", self.gpu_id) diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 3dd687a8b..a78710fa2 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -3,10 +3,10 @@ import asyncio import time import weakref -from collections.abc import AsyncGenerator, Iterable +from collections.abc import AsyncGenerator, Iterable, Sequence from dataclasses import asdict from pprint import pformat -from typing import Any +from typing import Any, cast from vllm.config import VllmConfig from vllm.inputs.preprocess import InputPreprocessor @@ -32,6 +32,7 @@ from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load from vllm_omni.entrypoints.utils import ( get_final_stage_id_for_e2e, ) +from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams, OmniTokensPrompt # Internal imports (our code) from vllm_omni.lora.request import LoRARequest @@ -66,10 +67,8 @@ class AsyncOmni(OmniBase): asynchronous LLM and Diffusion models. Args: - *args: Variable length argument list. - - args[0]: Model name or path to load. + model: Model name or path to load. **kwargs: Arbitrary keyword arguments. - - model: Model name or path to load (if not in args). - stage_configs_path: Optional path to YAML file containing stage configurations. If None, configurations are loaded from the model. - log_stats: Whether to enable statistics logging @@ -95,7 +94,7 @@ class AsyncOmni(OmniBase): ... print(output) """ - def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: + def __init__(self, model: str, **kwargs: dict[str, Any]) -> None: # Pause/resume control attributes self._pause_cond: asyncio.Condition = asyncio.Condition() self._paused: bool = False @@ -104,7 +103,7 @@ class AsyncOmni(OmniBase): self.request_states: dict[str, ClientRequestState] = {} self.output_handler: asyncio.Task | None = None - super().__init__(*args, **kwargs) + super().__init__(model, **kwargs) # Register weak reference cleanup (called on garbage collection) self._weak_finalizer = weakref.finalize( @@ -233,7 +232,14 @@ class AsyncOmni(OmniBase): if hasattr(self, "_weak_finalizer"): self._weak_finalizer() - async def generate(self, *args: Any, **kwargs: dict[str, Any]) -> AsyncGenerator[OmniRequestOutput, None]: + async def generate( + self, + prompt: OmniPromptType, + request_id: str, + sampling_params_list: Sequence[OmniSamplingParams] | None = None, + *, + output_modalities: list[str] | None = None, + ) -> AsyncGenerator[OmniRequestOutput, None]: """Generate outputs for the given prompt asynchronously. Coordinates multi-stage pipeline through YAML configuration. @@ -243,21 +249,13 @@ class AsyncOmni(OmniBase): sampling parameters from the sampling_params_list. Args: - *args: Arguments for generation. - - prompt: Prompt to process. Can be a text string, token IDs, - or multimodal prompt. - - request_id: Unique identifier for this request - - sampling_params_list: List of SamplingParams, one for each stage. - Must have the same length as the number of stages. - If None, uses default sampling params for each stage. - **kwargs: Additional arguments for generation. - - prompt: Prompt to process. Can be a text string, token IDs, - or multimodal prompt. - - request_id: Unique identifier for this request - - sampling_params_list: List of SamplingParams, one for each stage. - Must have the same length as the number of stages. - If None, uses default sampling params for each stage. - - output_modalities: Optional list of output modalities. + prompt: Prompt to process. Can be a text string, token IDs, + or multimodal prompt. + request_id: Unique identifier for this request + sampling_params_list: List of SamplingParams, one for each stage. + Must have the same length as the number of stages. + If None, uses default sampling params for each stage. + output_modalities: Optional list of output modalities. Yields: OmniRequestOutput objects as they are produced by each stage. @@ -276,33 +274,9 @@ class AsyncOmni(OmniBase): # Start output handler on the first call to generate() self._run_output_handler() - prompt = args[0] if args else kwargs.get("prompt") - request_id = args[1] if len(args) > 1 else kwargs.get("request_id") - sampling_params_list = args[2] if len(args) > 2 else kwargs.get("sampling_params_list") - output_modalities = kwargs.get("output_modalities", None) # TODO: lora_request, trace_headers, priority are not supported yet - if sampling_params_list is None: - # For Omni LLM, the params are parsed via the yaml file. For the current version, - # diffusion params can parsed via the command line. - omni_params_kwargs = { - k: v for k, v in kwargs.items() if k not in ["prompt", "request_id", "output_modalities"] - } - - per_stage_params: list[Any] = [] - for stage_id, stage in enumerate(self.stage_list): - stage_type = getattr(stage, "stage_type", "llm") - if stage_type == "diffusion": - default_dict = self.default_sampling_params_list[stage_id] - # Merge user-provided kwargs - merged = {**default_dict, **omni_params_kwargs} - # Diffusion only needs to keep diff params, will be used via OmniDiffusionRequest - per_stage_params.append(merged) - else: - # LLM directly constructs SamplingParams, don't use the merged params - per_stage_params.append(self.default_sampling_params_list[stage_id]) - - sampling_params_list = per_stage_params + sampling_params_list = self.default_sampling_params_list if len(sampling_params_list) != len(self.stage_list): raise ValueError(f"Expected {len(self.stage_list)} sampling params, got {len(sampling_params_list)}") @@ -334,11 +308,11 @@ class AsyncOmni(OmniBase): stage_queues = {stage_id: asyncio.Queue() for stage_id in range(num_stages)} req_state.stage_queues = stage_queues for i in range(num_stages): - sp: SamplingParams = sampling_params_list[i] - engine_inputs = prompt + sp: SamplingParams = cast(SamplingParams, sampling_params_list[i]) + engine_inputs = cast(OmniTokensPrompt, prompt) if i != 0: - prompt_token_ids = prompt["prompt_token_ids"] - prompt_1 = prompt.copy() + prompt_token_ids = engine_inputs["prompt_token_ids"] + prompt_1 = engine_inputs.copy() prompt_1["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids) prompt_1["multi_modal_data"] = prompt_1["mm_processor_kwargs"] = None engine_inputs = prompt_1 diff --git a/vllm_omni/entrypoints/async_omni_diffusion.py b/vllm_omni/entrypoints/async_omni_diffusion.py index 6d34e3445..535f04f7d 100644 --- a/vllm_omni/entrypoints/async_omni_diffusion.py +++ b/vllm_omni/entrypoints/async_omni_diffusion.py @@ -12,16 +12,15 @@ import asyncio import uuid from collections.abc import AsyncGenerator, Iterable from concurrent.futures import ThreadPoolExecutor -from dataclasses import fields from typing import Any -from PIL import Image from vllm.logger import init_logger from vllm.transformers_utils.config import get_hf_file_to_dict from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig from vllm_omni.diffusion.diffusion_engine import DiffusionEngine from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType from vllm_omni.lora.request import LoRARequest from vllm_omni.outputs import OmniRequestOutput @@ -109,68 +108,19 @@ class AsyncOmniDiffusion: logger.info("AsyncOmniDiffusion initialized with model: %s", model) - def _prepare_request( - self, - prompt: str, - request_id: str | None = None, - **kwargs: Any, - ) -> OmniDiffusionRequest: - """Prepare a diffusion request from prompt and parameters. - - Args: - prompt: Text prompt for image generation - request_id: Optional unique identifier for the request - **kwargs: Additional generation parameters - - Returns: - OmniDiffusionRequest ready for processing - """ - if request_id is None: - request_id = f"diff-{uuid.uuid4().hex[:16]}" - - field_names = {f.name for f in fields(OmniDiffusionRequest)} - - init_kwargs = { - "prompt": prompt, - "request_id": request_id, - } - - for key, value in kwargs.items(): - if key in field_names: - init_kwargs[key] = value - - if "guidance_scale" in kwargs: - init_kwargs["guidance_scale_provided"] = True - - return OmniDiffusionRequest(**init_kwargs) - async def generate( self, - prompt: str, + prompt: OmniPromptType, + sampling_params: OmniDiffusionSamplingParams, request_id: str | None = None, - num_inference_steps: int = 50, - guidance_scale: float | None = None, - height: int | None = None, - width: int | None = None, - negative_prompt: str | None = None, - num_outputs_per_prompt: int = 1, - seed: int | None = None, - lora_request=None, - **kwargs: Any, + lora_request: LoRARequest | None = None, ) -> OmniRequestOutput: """Generate images asynchronously from a text prompt. Args: prompt: Text prompt describing the desired image + sampling_params: Sampling parameters request_id: Optional unique identifier for tracking the request - num_inference_steps: Number of denoising steps (default: 50) - guidance_scale: Classifier-free guidance scale (optional, uses model defaults if omitted) - height: Optional image height in pixels - width: Optional image width in pixels - negative_prompt: Optional negative prompt for guidance - num_outputs_per_prompt: Number of images to generate (default: 1) - seed: Optional random seed for reproducibility - **kwargs: Additional generation parameters Returns: OmniRequestOutput containing generated images @@ -181,64 +131,38 @@ class AsyncOmniDiffusion: if request_id is None: request_id = f"diff-{uuid.uuid4().hex[:16]}" - # Prepare request - request_kwargs = { - "prompt": prompt, - "request_id": request_id, - "num_inference_steps": num_inference_steps, - "height": height, - "width": width, - "negative_prompt": negative_prompt, - "num_outputs_per_prompt": num_outputs_per_prompt, - "seed": seed, - "lora_request": lora_request, - **kwargs, - } - if guidance_scale is not None: - request_kwargs["guidance_scale"] = guidance_scale + if sampling_params.guidance_scale: + sampling_params.guidance_scale_provided = True - request = self._prepare_request(**request_kwargs) + if lora_request is not None: + sampling_params.lora_request = lora_request + + request = OmniDiffusionRequest( + prompts=[prompt], + sampling_params=sampling_params, + request_ids=[request_id], + ) logger.debug("Starting generation for request %s", request_id) # Run engine in thread pool loop = asyncio.get_event_loop() try: + # In async mode, only a single request is submitted at a time result = await loop.run_in_executor( self._executor, self.engine.step, - [request], + request, ) + result = result[0] except Exception as e: logger.error("Generation failed for request %s: %s", request_id, e) raise RuntimeError(f"Diffusion generation failed: {e}") from e - # Check if result is already OmniRequestOutput - if isinstance(result, OmniRequestOutput): - # Update request_id if needed - if not result.request_id: - result.request_id = request_id - return result - - # Process results if not OmniRequestOutput - images: list[Image.Image] = [] - if result is not None: - if isinstance(result, list): - for item in result: - if isinstance(item, Image.Image): - images.append(item) - elif isinstance(result, Image.Image): - images.append(result) - - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=images, - prompt=prompt, - metrics={ - "num_inference_steps": num_inference_steps, - "guidance_scale": request.guidance_scale, - }, - ) + # Update request_id if needed + if not result.request_id: + result.request_id = request_id + return result async def generate_stream( self, diff --git a/vllm_omni/entrypoints/omni.py b/vllm_omni/entrypoints/omni.py index 90b344c6f..97357dc3b 100644 --- a/vllm_omni/entrypoints/omni.py +++ b/vllm_omni/entrypoints/omni.py @@ -10,14 +10,13 @@ from collections.abc import Callable, Generator, Sequence from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import asdict from pprint import pformat -from typing import Any +from typing import Any, Literal, overload from omegaconf import OmegaConf from tqdm.auto import tqdm -from vllm.inputs import PromptType +from vllm import SamplingParams from vllm.logger import init_logger -from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.distributed.omni_connectors import ( get_stage_connector_config, initialize_orchestrator_connectors, @@ -42,6 +41,7 @@ from vllm_omni.entrypoints.utils import ( load_stage_configs_from_yaml, resolve_model_config_path, ) +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType, OmniSamplingParams from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) @@ -82,10 +82,8 @@ class OmniBase: """Base class for serving Omni models. Args: - *args: Variable length argument list. - - args[0]: Model name or path to load. + model: Model name or path to load. **kwargs: Arbitrary keyword arguments. - - model: Model name or path to load (if not in args). - stage_configs_path: Optional path to YAML file containing stage configurations. If None, configurations are loaded from the model. - log_stats: Whether to enable statistics logging @@ -102,14 +100,9 @@ class OmniBase: - Additional keyword arguments passed to stage engines. """ - def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: - model = args[0] if args else kwargs.get("model", "") - assert model != "", "Null model id detected, please specify a model id." + def __init__(self, model: str, **kwargs: Any) -> None: model = omni_snapshot_download(model) - if args: - args[0] = model - elif kwargs.get("model", "") != "": - kwargs["model"] = model + kwargs["model"] = model # Stage management attributes self.stage_list: list[OmniStage] = [] @@ -502,10 +495,8 @@ class Omni(OmniBase): """Unified entrypoint for both LLM and Diffusion models for better usability. Args: - *args: Variable length argument list. - - args[0]: Model name or path to load. + model: Model name or path to load. **kwargs: Arbitrary keyword arguments. - - model: Model name or path to load (if not in args). - stage_configs_path: Optional path to YAML file containing stage configurations. If None, configurations are loaded from the model. - log_stats: Whether to enable statistics logging @@ -527,8 +518,8 @@ class Omni(OmniBase): >>> print(outputs) """ - def __init__(self, *args: Any, **kwargs: dict[str, Any]) -> None: - super().__init__(*args, **kwargs) + def __init__(self, model: str, **kwargs: Any) -> None: + super().__init__(model, **kwargs) # Register weak reference cleanup (called on garbage collection) self._weak_finalizer = weakref.finalize( @@ -539,8 +530,31 @@ class Omni(OmniBase): self._ray_pg, ) + @overload def generate( - self, *args: Any, **kwargs: dict[str, Any] + self, + prompts: OmniPromptType | Sequence[OmniPromptType], + sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None, + *, + py_generator: Literal[True], + ) -> Generator[OmniRequestOutput, None, None]: ... + + @overload + def generate( + self, + prompts: OmniPromptType | Sequence[OmniPromptType], + sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None, + *, + py_generator: Literal[False] = False, + ) -> list[OmniRequestOutput]: ... + + def generate( + self, + prompts: OmniPromptType | Sequence[OmniPromptType], + sampling_params_list: OmniSamplingParams | Sequence[OmniSamplingParams] | None = None, + *, + py_generator: bool = False, + use_tqdm: bool | Callable[..., tqdm] = True, ) -> Generator[OmniRequestOutput, None, None] | list[OmniRequestOutput]: """Generate outputs for the given prompts. @@ -548,12 +562,10 @@ class Omni(OmniBase): Each stage will use OmniLLM or OmniDiffusion based on stage_type. Args: - *args: Variable length argument list. - - args[0]: Input prompts for generation. - - args[1]: Optional list of per-stage parameters. - **kwargs: Arbitrary keyword arguments. - - prompt: Input prompts for generation (if not in args). - - sampling_params_list: Optional list of per-stage parameters (if not in args). + prompts: Input prompt(s) for generation. + sampling_params_list: Optional list of per-stage parameters. + py_generator: Whether the returned result(s) are wrapped in a generator instead of a list. + use_tqdm: Whether to use tqdm progress bar Returns: List of OmniRequestOutput objects, one for each input prompt. @@ -563,40 +575,26 @@ class Omni(OmniBase): Raises: ValueError: If sampling_params_list is None or has incorrect length. """ - prompts = args[0] if args else kwargs.get("prompts") - sampling_params_list = args[1] if len(args) > 1 else kwargs.get("sampling_params_list") - py_generator = kwargs.get("py_generator", False) - if prompts is None: - if kwargs.get("prompt") is None: - raise ValueError("prompts is required for generation") - prompts = kwargs.get("prompt") - if sampling_params_list is None: - # For Omni LLM, the params are parsed via the yaml file. For the current version, - # diffusion params can parsed via the command line. - omni_params_kwargs = { - k: v for k, v in kwargs.items() if k not in ["prompt", "request_id", "output_modalities"] - } - - per_stage_params: list[Any] = [] - for stage_id, stage in enumerate(self.stage_list): - stage_type = getattr(stage, "stage_type", "llm") - if stage_type == "diffusion": - default_dict = self.default_sampling_params_list[stage_id] - # Merge user-provided kwargs - merged = {**default_dict, **omni_params_kwargs} - # Diffusion only needs to keep diff params, will be used via OmniDiffusionRequest - per_stage_params.append(merged) + sampling_params_list = self.default_sampling_params_list + elif not isinstance(sampling_params_list, Sequence): + # TODO: After the recent introduction of BAGEL model (one LLM and one Diffusion), + # expect the text_to_image example code to run when only passing one OmniDiffusionSamplingParams + # This behavior may be confusing, and future PR can improve it. + per_stage_params: list[OmniSamplingParams] = [] + for default_stage_sp in self.default_sampling_params_list: + default_sp_type = default_stage_sp.__class__ + if default_sp_type == sampling_params_list.__class__: + per_stage_params.append(sampling_params_list) else: - # LLM directly constructs SamplingParams, don't use the merged params - per_stage_params.append(self.default_sampling_params_list[stage_id]) - + per_stage_params.append(default_stage_sp) sampling_params_list = per_stage_params + try: if py_generator: return self._run_generation_with_generator(prompts, sampling_params_list) else: - outputs = list(self._run_generation(prompts, sampling_params_list)) + outputs = list(self._run_generation(prompts, sampling_params_list, use_tqdm)) return outputs except Exception as e: logger.exception("[Orchestrator] Failed to run generation: %s", e) @@ -606,8 +604,8 @@ class Omni(OmniBase): def _run_generation_with_generator( self, - prompts: PromptType | Sequence[PromptType] | OmniDiffusionRequest | Sequence[OmniDiffusionRequest], - sampling_params_list: Any | Sequence[Any] | None, + prompts: OmniPromptType | Sequence[OmniPromptType], + sampling_params_list: Sequence[OmniSamplingParams], ) -> Generator[OmniRequestOutput, None, None]: """Run generation through all stages in the pipeline and return a generator.""" gen = self._run_generation(prompts, sampling_params_list) @@ -622,8 +620,8 @@ class Omni(OmniBase): def _run_generation( self, - prompts: PromptType | Sequence[PromptType] | OmniDiffusionRequest | Sequence[OmniDiffusionRequest], - sampling_params_list: Any | Sequence[Any] | None = None, + prompts: OmniPromptType | Sequence[OmniPromptType], + sampling_params_list: Sequence[OmniSamplingParams], use_tqdm: bool | Callable[..., tqdm] = True, ) -> Generator[OmniRequestOutput, None, None]: """Run generation through all stages in the pipeline.""" @@ -631,18 +629,20 @@ class Omni(OmniBase): if sampling_params_list is None: raise ValueError("sampling_params_list is required for pipelined generation") - # Normalize sampling_params_list to a list - if not isinstance(sampling_params_list, (list, tuple)): - sampling_params_list = [sampling_params_list] - else: - sampling_params_list = list(sampling_params_list) - if len(sampling_params_list) != len(self.stage_list): raise ValueError(f"Expected {len(self.stage_list)} sampling params, got {len(sampling_params_list)}") + for i, (stage, sp) in enumerate(zip(self.stage_list, sampling_params_list)): + ExpectedSPType = OmniDiffusionSamplingParams if stage.stage_type == "diffusion" else SamplingParams + if not isinstance(sp, ExpectedSPType): + raise ValueError( + f"Expected sampling parameters with type {ExpectedSPType} in stage {i}, got {sp.__class__}" + ) + # Normalize prompts to a list for per-request iteration - if not isinstance(prompts, (list, tuple)): - request_prompts: list[PromptType] = [prompts] + # str is also Sequence but only test list-like containers here + if isinstance(prompts, str) or not isinstance(prompts, Sequence): + request_prompts: list[OmniPromptType] = [prompts] else: request_prompts = list(prompts) @@ -650,8 +650,8 @@ class Omni(OmniBase): num_stages = len(self.stage_list) # Generate globally unique request IDs and map them to original prompts - request_ids: list[str] = [f"{i}_{uuid.uuid4()}" for i in range(len(request_prompts))] - request_id_to_prompt: dict[str, PromptType] = {rid: p for rid, p in zip(request_ids, request_prompts)} + request_ids = [f"{i}_{uuid.uuid4()}" for i in range(len(request_prompts))] + request_id_to_prompt = {rid: p for rid, p in zip(request_ids, request_prompts)} # Track per-request start time for end-to-end timing _req_start_ts: dict[str, float] = {} diff --git a/vllm_omni/entrypoints/omni_diffusion.py b/vllm_omni/entrypoints/omni_diffusion.py index f5b0a0a40..5ad9a91c8 100644 --- a/vllm_omni/entrypoints/omni_diffusion.py +++ b/vllm_omni/entrypoints/omni_diffusion.py @@ -3,7 +3,7 @@ import logging import uuid -from dataclasses import fields +from collections.abc import Sequence from vllm.logger import init_logger from vllm.transformers_utils.config import get_hf_file_to_dict @@ -11,6 +11,8 @@ from vllm.transformers_utils.config import get_hf_file_to_dict from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig from vllm_omni.diffusion.diffusion_engine import DiffusionEngine from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType +from vllm_omni.outputs import OmniRequestOutput # TODO configure logging properly logging.basicConfig(level=logging.INFO) @@ -18,21 +20,6 @@ logging.basicConfig(level=logging.INFO) logger = init_logger(__name__) -def prepare_requests(prompt: str | list[str], **kwargs): - field_names = {f.name for f in fields(OmniDiffusionRequest)} - - init_kwargs = {"prompt": prompt} - - for key, value in kwargs.items(): - if key in field_names: - init_kwargs[key] = value - - if "guidance_scale" in kwargs: - init_kwargs["guidance_scale_provided"] = True - - return OmniDiffusionRequest(**init_kwargs) - - class OmniDiffusion: """ It is the main class to interact with vLLM-Omni diffusion models. @@ -100,42 +87,24 @@ class OmniDiffusion: def generate( self, - prompt: str | list[str], - **kwargs, - ): - prompts = [] - if isinstance(prompt, str): - prompts.append(prompt) - elif isinstance(prompt, list): - prompts.extend(prompt) + prompts: OmniPromptType | Sequence[OmniPromptType], + sampling_params: OmniDiffusionSamplingParams, + request_ids: list[str] = [], + ) -> list[OmniRequestOutput]: + if isinstance(prompts, (str, dict)): + prompts = [prompts] else: - raise ValueError("Prompt must be a string or a list of strings") - - requests: list[OmniDiffusionRequest] = [] + prompts = list(prompts) # Check if request_id is provided in kwargs - request_id = kwargs.get("request_id") - request_ids = kwargs.pop("request_ids", None) + if len(request_ids) < len(prompts): + request_ids.extend(f"{i + len(request_ids)}_{uuid.uuid4()}" for i in range(len(prompts) - len(request_ids))) - for i, p in enumerate(prompts): - req_kwargs = kwargs.copy() - if request_ids and isinstance(request_ids, list) and i < len(request_ids): - req_kwargs["request_id"] = request_ids[i] - elif request_id is None: - # Generate default ID consistent with OmniLLM: "{i}_{uuid}" - req_kwargs["request_id"] = f"{i}_{uuid.uuid4()}" + request = OmniDiffusionRequest(prompts, sampling_params, request_ids) + return self._run_engine(request) - requests.append( - prepare_requests( - p, - **req_kwargs, - ) - ) - logger.info(f"Prepared {len(requests)} requests for generation.") - return self._run_engine(requests) - - def _run_engine(self, requests: list[OmniDiffusionRequest]): - return self.engine.step(requests) + def _run_engine(self, request: OmniDiffusionRequest) -> list[OmniRequestOutput]: + return self.engine.step(request) def close(self) -> None: self.engine.close() diff --git a/vllm_omni/entrypoints/omni_stage.py b/vllm_omni/entrypoints/omni_stage.py index e0e699c9c..0a5ee55be 100644 --- a/vllm_omni/entrypoints/omni_stage.py +++ b/vllm_omni/entrypoints/omni_stage.py @@ -15,9 +15,11 @@ import queue import sys import time import traceback +from collections.abc import Sequence from dataclasses import fields -from typing import Any +from typing import Any, Literal, cast +from vllm import PromptType, RequestOutput from vllm.inputs import TextPrompt from vllm.inputs.preprocess import InputPreprocessor from vllm.logger import init_logger @@ -31,6 +33,7 @@ from vllm.v1.engine.llm_engine import LLMEngine from vllm_omni.diffusion.data import OmniDiffusionConfig from vllm_omni.distributed.omni_connectors import build_stage_connectors from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector +from vllm_omni.distributed.omni_connectors.connectors.base import OmniConnectorBase from vllm_omni.distributed.ray_utils.utils import kill_ray_actor, start_ray_actor from vllm_omni.engine.arg_utils import AsyncOmniEngineArgs from vllm_omni.entrypoints.async_omni_diffusion import AsyncOmniDiffusion @@ -46,7 +49,8 @@ from vllm_omni.entrypoints.stage_utils import ( maybe_dump_to_shm, set_stage_devices, ) -from vllm_omni.inputs.data import OmniTokensPrompt +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType, OmniSamplingParams, OmniTokensPrompt +from vllm_omni.outputs import OmniRequestOutput from vllm_omni.utils import detect_device_type logger = init_logger(__name__) @@ -64,35 +68,6 @@ def _build_od_config(engine_args: dict[str, Any], model: str) -> dict[str, Any]: return od_config -def prepare_sampling_params(sampling_params: Any, stage_type: str) -> Any: - """Prepare sampling parameters for the given stage type. - - Args: - sampling_params: Raw sampling parameters (dict or SamplingParams) - stage_type: Either "llm" or "diffusion" - - Returns: - Processed sampling parameters ready for engine consumption - """ - if stage_type == "diffusion": - # For diffusion stages: extract kwargs, handling different input types - if isinstance(sampling_params, dict): - diffusion_kwargs = dict(sampling_params) - else: - diffusion_kwargs = getattr(sampling_params, "__dict__", {}) or {} - - # Remove 'prompt' and 'request_id' to avoid conflict with explicit arguments - diffusion_kwargs.pop("prompt", None) - diffusion_kwargs.pop("request_id", None) - return diffusion_kwargs - - else: # stage_type == "llm" - # For LLM stages: ensure we have a SamplingParams object - if isinstance(sampling_params, dict): - return SamplingParams(**sampling_params) - return sampling_params - - class OmniStage: """Stage manager for orchestrating a single stage in the omni pipeline. @@ -123,7 +98,7 @@ class OmniStage: self.engine_outputs = None self.is_comprehension = getattr(stage_config, "is_comprehension", False) # Support for different stage types: "llm" (default) or "diffusion" - self.stage_type = getattr(stage_config, "stage_type", "llm") + self.stage_type: Literal["llm", "diffusion"] = getattr(stage_config, "stage_type", "llm") if hasattr(stage_config, "custom_process_input_func"): # Import the module specified in the config (already a full module path) module_path, func_name = stage_config.custom_process_input_func.rsplit(".", 1) @@ -137,7 +112,14 @@ class OmniStage: default_sampling_params = getattr(stage_config, "default_sampling_params", {}) # For LLM stage, this can directly be a SamplingParams-compatible dict; # For diffusion stage, this only serves as default values for diffusion kwargs. - self.default_sampling_params = _to_dict(default_sampling_params) + default_sampling_params = _to_dict(default_sampling_params) + # Further convert it to dataclass to check fields + try: + self.default_sampling_params = ( + SamplingParams if self.stage_type == "llm" else OmniDiffusionSamplingParams + )(**default_sampling_params) + except TypeError as error: + raise TypeError(f"Invalid default_sampling_params for stage {self.stage_id}: {error}") from error # Runtime orchestration state (added) self._in_q: mp.Queue | None = None self._out_q: mp.Queue | None = None @@ -519,7 +501,7 @@ def _stage_worker( runtime_cfg = stage_payload.get("runtime", {}) shm_threshold_bytes = int(stage_payload.get("shm_threshold_bytes", 65536)) connectors_config = stage_payload.get("connectors_config", {}) - stage_type = stage_payload.get("stage_type", "llm") + stage_type: Literal["llm", "diffusion"] = stage_payload.get("stage_type", "llm") # Aggregates for running average _agg_total_tokens = 0 @@ -708,15 +690,14 @@ def _stage_worker( pass logger.debug("Engine initialized") # Initialize OmniConnectors if configured - connectors = {} + connectors: dict[tuple[str, str], OmniConnectorBase] | None = {} if connectors_config: - built_connectors = build_stage_connectors( + connectors = build_stage_connectors( stage_id=stage_id, connectors_config=connectors_config, ) - if built_connectors is None: + if connectors is None: return - connectors = built_connectors # Signal readiness to orchestrator try: @@ -785,6 +766,7 @@ def _stage_worker( continue batch_tasks: list[dict[str, Any]] = [task] + tasks_failed_to_add_to_batch: list[dict[str, Any]] = [] start_time = _time.time() if max_batch_size > 1: while len(batch_tasks) < max_batch_size: @@ -800,7 +782,20 @@ def _stage_worker( if extra_type == OmniStageTaskType.PROFILER_STOP: out_q.put({"type": "profiler_result", "data": p_data}) continue - batch_tasks.append(extra) + # Ensure that all tasks have the same sampling params + # If no, put them in a temporary container and add back to queue + # This should be always true, because user only calls omni.generate() once and it blocks + # User can only pass one sampling param object, but the list of prompts are separated. + if task.get("sampling_params") != extra.get("sampling_params"): + logger.warning( + """In offline mode, expect all prompts in one `omni.generate()` call to share same sampling params""" # noqa: E501 # line too long + f"""However, prompt {task.get("engine_inputs")} has sampling params {task.get("sampling_params")}, """ # noqa: E501 # line too long + f"""whereas the prompt {extra.get("engine_inputs")} has sampling params {extra.get("sampling_params")}.""" # noqa: E501 # line too long + """The two tasks cannot be combined in one batch request.""" + ) + tasks_failed_to_add_to_batch.append(extra) + else: + batch_tasks.append(extra) end_time = _time.time() duration = end_time - start_time if duration > batch_timeout: @@ -815,9 +810,13 @@ def _stage_worker( break else: continue + for task_to_readd in tasks_failed_to_add_to_batch: + in_q.put(task_to_readd) + # Ensure that the popped tasks are with identical sampling params. Take one of them. + batch_engine_sampling_params: OmniSamplingParams = batch_tasks[0]["sampling_params"] batch_request_ids: list[Any] = [] - batch_engine_inputs: list[Any] = [] + batch_engine_inputs: list[OmniPromptType] = [] _rx_bytes_by_rid: dict[Any, int] = {} _rx_decode_ms_by_rid: dict[Any, float] = {} _in_flight_ms_by_rid: dict[Any, float] = {} @@ -839,6 +838,9 @@ def _stage_worker( connectors=connectors, stage_id=stage_id, ) + # TODO: hack type annotation for now. + # A better way is to refine type annotation of connection and task/payloads, maybe using template types. + ein = cast(OmniPromptType | Sequence[OmniPromptType] | None, ein) if ein is None or _rx_metrics is None: raise RuntimeError( @@ -850,17 +852,14 @@ def _stage_worker( _rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0)) batch_request_ids.append(rid) - if isinstance(ein, list): + if isinstance(ein, (str, dict)): + # Types like OmniTextPrompt, TextPrompt are TypedDict, essentially dict and enters this branch + batch_engine_inputs.append(ein) + elif isinstance(ein, Sequence): batch_engine_inputs.extend(ein) - elif isinstance(ein, dict): - batch_engine_inputs.append(ein) - elif isinstance(ein, str): - # For diffusion stage-0, ein might be a string prompt directly - batch_engine_inputs.append(ein) else: - # For other types (e.g., OmniTokensPrompt, TextPrompt), append as-is + # Other unknown types, append as-is batch_engine_inputs.append(ein) - sampling_params = batch_tasks[0]["sampling_params"] logger.debug( "Received batch size=%d, request_ids=%s", len(batch_tasks), @@ -868,65 +867,30 @@ def _stage_worker( ) try: _batch_seq += 1 - gen_outputs: list[Any] = [] + gen_outputs: list[OmniRequestOutput | RequestOutput] = [] _gen_t0 = _time.time() if stage_type == "diffusion": - # For diffusion, batch_engine_inputs should be prompts (strings) - # Convert to list of strings if needed - prompts = [] - for ein in batch_engine_inputs: - if isinstance(ein, str): - prompts.append(ein) - elif isinstance(ein, dict) and "prompt" in ein: - prompts.append(ein["prompt"]) - elif hasattr(ein, "prompt"): - prompts.append(ein.prompt) - else: - prompts.append(str(ein)) - # Prepare diffusion kwargs from sampling parameters - diffusion_kwargs = prepare_sampling_params(sampling_params, "diffusion") - - # Pass batch_request_ids to ensure correct ID mapping - diffusion_kwargs["request_ids"] = batch_request_ids - + stage_engine = cast(OmniDiffusion, stage_engine) + batch_engine_sampling_params = cast(OmniDiffusionSamplingParams, batch_engine_sampling_params) # Diffusion generate returns results directly, not an iterator - diffusion_results = stage_engine.generate(prompts, **diffusion_kwargs) - # Convert to list format compatible with LLM outputs - # Ensure each result has a request_id for proper mapping - if isinstance(diffusion_results, list): - gen_outputs = diffusion_results - # Assign request_ids if not present - for idx, result in enumerate(gen_outputs): - if not hasattr(result, "request_id") or result.request_id is None: - if idx < len(batch_request_ids): - if hasattr(result, "request_id"): - result.request_id = batch_request_ids[idx] - else: - # Create a wrapper object if result doesn't support request_id - from types import SimpleNamespace - - wrapped = SimpleNamespace() - wrapped.request_id = batch_request_ids[idx] - wrapped.output = result - gen_outputs[idx] = wrapped - else: - gen_outputs = [diffusion_results] - # Assign request_id to single result - if len(batch_request_ids) > 0: - if hasattr(gen_outputs[0], "request_id"): - gen_outputs[0].request_id = batch_request_ids[0] - else: - from types import SimpleNamespace - - wrapped = SimpleNamespace() - wrapped.request_id = batch_request_ids[0] - wrapped.output = gen_outputs[0] - gen_outputs[0] = wrapped + diffusion_results = stage_engine.generate( + batch_engine_inputs, batch_engine_sampling_params, batch_request_ids + ) + gen_outputs.extend(diffusion_results) + # Assign request_ids if not present + for idx, result in enumerate(gen_outputs): + if not hasattr(result, "request_id") or result.request_id is None: + if idx < len(batch_request_ids): + result.request_id = batch_request_ids[idx] else: - # LLM engine: use vLLM native SamplingParams - llm_sampling_params = prepare_sampling_params(sampling_params, "llm") - for ro in stage_engine.generate(batch_engine_inputs, llm_sampling_params, use_tqdm=False): - gen_outputs.append(ro) + stage_engine = cast(OmniLLM, stage_engine) + batch_engine_sampling_params = cast(SamplingParams, batch_engine_sampling_params) + results = stage_engine.generate( + batch_engine_inputs, # type: ignore # silent complaints about list of subclassed TypedDict + batch_engine_sampling_params, + use_tqdm=False, + ) + gen_outputs.extend(results) _gen_t1 = _time.time() _gen_ms = (_gen_t1 - _gen_t0) * 1000.0 logger.debug(f"Generate done: batch={len(batch_tasks)}, req_ids={batch_request_ids}, gen_ms={_gen_ms:.1f}") @@ -935,7 +899,7 @@ def _stage_worker( req_to_outputs: dict[Any, list[Any]] = {rid: [] for rid in batch_request_ids} unmapped: list[Any] = [] for ro in gen_outputs: - rid = getattr(ro, "request_id", None) + rid = ro.request_id if rid in req_to_outputs: req_to_outputs[rid].append(ro) else: @@ -1354,6 +1318,10 @@ async def _stage_worker_async( connectors=connectors, stage_id=stage_id, ) + # TODO: hack type annotation for now. + # A better way is to refine type annotation of connection and task/payloads, maybe using template types. + ein = cast(OmniPromptType | Sequence[OmniPromptType] | None, ein) + if ein is None or _rx_metrics is None: raise RuntimeError( f"[Stage-{stage_id}] Missing connector payload for request {rid}. " @@ -1362,36 +1330,23 @@ async def _stage_worker_async( _rx_decode_ms_by_rid[rid] = float(_rx_metrics.get("rx_decode_time_ms", 0.0)) _rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0)) - sampling_params = task["sampling_params"] logger.debug("Received batch size=1, request_ids=%s", rid) _gen_t0 = _time.time() - if isinstance(ein, list): + if isinstance(ein, Sequence) and not isinstance(ein, str): ein = ein[0] if stage_type == "diffusion": - # For diffusion, ein should be prompts (strings) - # Convert to string if needed - if isinstance(ein, str): - prompt = ein - elif isinstance(ein, dict) and "prompt" in ein: - prompt = ein["prompt"] - elif hasattr(ein, "prompt"): - prompt = ein.prompt - else: - prompt = str(ein) - - # Prepare diffusion kwargs from sampling parameters - diffusion_kwargs = prepare_sampling_params(sampling_params, "diffusion") + diffusion_sampling_params = cast(OmniDiffusionSamplingParams, task["sampling_params"]) # AsyncOmniDiffusion.generate returns a single result, not an async generator - gen_output = await stage_engine.generate(prompt=prompt, request_id=rid, **diffusion_kwargs) + gen_output = await cast(AsyncOmniDiffusion, stage_engine).generate(ein, diffusion_sampling_params, rid) _gen_t1 = _time.time() _gen_ms = (_gen_t1 - _gen_t0) * 1000.0 await generation_out_q.put((rid, gen_output, _gen_ms)) else: - # LLM stages: ensure using SamplingParams - llm_sampling_params = prepare_sampling_params(sampling_params, "llm") + ein = cast(PromptType, ein) + llm_sampling_params: SamplingParams = task["sampling_params"] gen_output = None - async for res in stage_engine.generate(ein, llm_sampling_params, rid): + async for res in cast(AsyncLLM, stage_engine).generate(ein, llm_sampling_params, rid): gen_output = res _gen_t1 = _time.time() _gen_ms = (_gen_t1 - _gen_t0) * 1000.0 diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index a47a864cd..f80fe95fa 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -10,13 +10,14 @@ from argparse import Namespace from collections.abc import AsyncIterator from contextlib import asynccontextmanager from http import HTTPStatus -from typing import Any +from typing import Any, cast import vllm.envs as envs from fastapi import Depends, HTTPException, Request from fastapi.responses import JSONResponse, StreamingResponse from starlette.datastructures import State from starlette.routing import Route +from vllm import SamplingParams from vllm.engine.protocol import EngineClient from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages from vllm.entrypoints.launcher import serve_http @@ -80,6 +81,7 @@ from vllm_omni.entrypoints.openai.protocol.images import ( ) from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams, OmniTextPrompt from vllm_omni.lora.request import LoRARequest from vllm_omni.lora.utils import stable_lora_int_id @@ -865,7 +867,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request) HTTPException: For validation errors, missing engine, or generation failures """ # Get engine client (AsyncOmni) from app state - engine_client: EngineClient | None = getattr(raw_request.app.state, "engine_client", None) + engine_client: EngineClient | AsyncOmni | None = getattr(raw_request.app.state, "engine_client", None) if engine_client is None or not hasattr(engine_client, "stage_list"): raise HTTPException( status_code=HTTPStatus.SERVICE_UNAVAILABLE.value, @@ -925,10 +927,8 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request) try: # Build params - pass through user values directly - gen_params = { - "prompt": request.prompt, - "num_outputs_per_prompt": request.n, - } + prompt: OmniTextPrompt = {"prompt": request.prompt} + gen_params = OmniDiffusionSamplingParams(num_outputs_per_prompt=request.n) # Parse per-request LoRA (compatible with chat's extra_body.lora shape). if request.lora is not None: @@ -960,62 +960,72 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request) detail="Invalid lora object: both name and path are required.", ) - gen_params["lora_request"] = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)) + gen_params.lora_request = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)) if lora_scale is not None: - gen_params["lora_scale"] = float(lora_scale) + gen_params.lora_scale = float(lora_scale) # Parse and add size if provided if request.size: width, height = parse_size(request.size) - gen_params["height"] = height - gen_params["width"] = width + gen_params.height = height + gen_params.width = width size_str = f"{width}x{height}" else: size_str = "model default" # Add optional parameters ONLY if provided if request.num_inference_steps is not None: - gen_params["num_inference_steps"] = request.num_inference_steps + gen_params.num_inference_steps = request.num_inference_steps if request.negative_prompt is not None: - gen_params["negative_prompt"] = request.negative_prompt + prompt["negative_prompt"] = request.negative_prompt if request.guidance_scale is not None: - gen_params["guidance_scale"] = request.guidance_scale + gen_params.guidance_scale = request.guidance_scale if request.true_cfg_scale is not None: - gen_params["true_cfg_scale"] = request.true_cfg_scale + gen_params.true_cfg_scale = request.true_cfg_scale if request.seed is not None: - gen_params["seed"] = request.seed - gen_params["request_id"] = f"img_gen_{int(time.time())}" + gen_params.seed = request.seed + request_id = f"img_gen_{int(time.time())}" logger.info(f"Generating {request.n} image(s) {size_str}") # Generate images using AsyncOmni (multi-stage mode) + engine_client = cast(AsyncOmni, engine_client) result = None stage_list = getattr(engine_client, "stage_list", None) if isinstance(stage_list, list): - default_params_list = getattr(engine_client, "default_sampling_params_list", None) + default_params_list: list[OmniSamplingParams] | None = getattr( + engine_client, "default_sampling_params_list", None + ) if not isinstance(default_params_list, list): - default_params_list = [{} for _ in stage_types] + default_params_list = [ + OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types + ] else: default_params_list = list(default_params_list) if len(default_params_list) != len(stage_types): - default_params_list = (default_params_list + [{} for _ in stage_types])[: len(stage_types)] + default_params_list = ( + default_params_list + + [OmniDiffusionSamplingParams() if st == "diffusion" else SamplingParams() for st in stage_types] + )[: len(stage_types)] - sampling_params_list: list[dict[str, Any]] = [] + sampling_params_list: list[OmniSamplingParams] = [] for idx, stage_type in enumerate(stage_types): if stage_type == "diffusion": sampling_params_list.append(gen_params) else: base_params = default_params_list[idx] - sampling_params_list.append(dict(base_params) if isinstance(base_params, dict) else base_params) + sampling_params_list.append(base_params) async for output in engine_client.generate( - prompt=gen_params["prompt"], - request_id=gen_params["request_id"], + prompt=prompt, + request_id=request_id, sampling_params_list=sampling_params_list, ): result = output else: - result = await engine_client.generate(**gen_params) + result = await engine_client.generate( + prompt=prompt, request_id=request_id, sampling_params_list=[gen_params] + ) if result is None: raise HTTPException( diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index f35008260..db8c972a1 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -6,13 +6,16 @@ import uuid from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence from datetime import datetime, timedelta, timezone from io import BytesIO -from typing import TYPE_CHECKING, Any, Final, Optional +from typing import TYPE_CHECKING, Any, Final, Optional, cast import jinja2 from fastapi import Request from PIL import Image from pydantic import TypeAdapter +from vllm_omni.entrypoints.async_omni import AsyncOmni +from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt + try: import soundfile except ImportError: @@ -274,16 +277,11 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): lora_request=lora_request, ) - trace_headers = None if raw_request is None else await self._get_trace_headers(raw_request.headers) - generator = self.engine_client.generate( prompt=engine_prompt, request_id=request_id, sampling_params_list=sampling_params_list, output_modalities=output_modalities, - lora_request=lora_request, - trace_headers=trace_headers, - priority=request.priority, ) generators.append(generator) @@ -1904,29 +1902,30 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): logger.warning("Failed to decode reference image: %s", e) # Build generation kwargs - gen_kwargs: dict[str, Any] = { + gen_prompt: OmniTextPrompt = { "prompt": prompt, - "request_id": request_id, - "num_inference_steps": num_inference_steps, - "height": height, - "width": width, "negative_prompt": negative_prompt, - "num_outputs_per_prompt": num_outputs_per_prompt, - "seed": seed, } + gen_params = OmniDiffusionSamplingParams( + num_inference_steps=num_inference_steps, + height=height, + width=width, + num_outputs_per_prompt=num_outputs_per_prompt, + seed=seed, + ) if guidance_scale is not None: - gen_kwargs["guidance_scale"] = guidance_scale + gen_params.guidance_scale = guidance_scale # Add Qwen-Image specific parameter if true_cfg_scale is not None: - gen_kwargs["true_cfg_scale"] = true_cfg_scale + gen_params.true_cfg_scale = true_cfg_scale # Add video generation parameters if set if num_frames is not None: - gen_kwargs["num_frames"] = num_frames + gen_params.num_frames = num_frames if guidance_scale_2 is not None: - gen_kwargs["guidance_scale_2"] = guidance_scale_2 + gen_params.guidance_scale_2 = guidance_scale_2 # Parse per-request LoRA (works for both AsyncOmniDiffusion and AsyncOmni). if lora_body and isinstance(lora_body, dict): @@ -1949,16 +1948,17 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): lora_int_id = stable_lora_int_id(str(lora_path)) if lora_name and lora_path: lora_req = LoRARequest(str(lora_name), int(lora_int_id), str(lora_path)) - gen_kwargs["lora_request"] = lora_req + gen_params.lora_request = lora_req if lora_scale is not None: - gen_kwargs["lora_scale"] = float(lora_scale) + gen_params.lora_scale = float(lora_scale) except Exception as e: # pragma: no cover - safeguard logger.warning("Failed to parse LoRA request: %s", e) # Add reference image if provided if pil_images: if len(pil_images) == 1: - gen_kwargs["pil_image"] = pil_images[0] + gen_prompt["multi_modal_data"] = {} + gen_prompt["multi_modal_data"]["image"] = pil_images[0] else: od_config = getattr(self._diffusion_engine, "od_config", None) supports_multimodal_inputs = getattr(od_config, "supports_multimodal_inputs", False) @@ -1966,7 +1966,8 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # TODO: entry is asyncOmni. We hack the od config here. supports_multimodal_inputs = True if supports_multimodal_inputs: - gen_kwargs["pil_image"] = pil_images + gen_prompt["multi_modal_data"] = {} + gen_prompt["multi_modal_data"]["image"] = pil_images else: return self._create_error_response( "Multiple input images are not supported by the current diffusion model. " @@ -1979,18 +1980,24 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): # Handle both AsyncOmniDiffusion (returns OmniRequestOutput) and AsyncOmni (returns AsyncGenerator) if hasattr(self._diffusion_engine, "stage_list"): # AsyncOmni: iterate through async generator to get final output + diffusion_engine = cast(AsyncOmni, self._diffusion_engine) result = None - async for output in self._diffusion_engine.generate( - prompt=gen_kwargs["prompt"], - request_id=gen_kwargs.get("request_id"), - sampling_params_list=[gen_kwargs], # Pass as single-stage params + async for output in diffusion_engine.generate( + prompt=gen_prompt, + sampling_params_list=[gen_params], # Pass as single-stage params + request_id=request_id, ): result = output if result is None: return self._create_error_response("No output generated from AsyncOmni") else: # AsyncOmniDiffusion: direct call - result = await self._diffusion_engine.generate(**gen_kwargs) + diffusion_engine = cast(AsyncOmniDiffusion, self._diffusion_engine) + result = await diffusion_engine.generate( + prompt=gen_prompt, + sampling_params=gen_params, + request_id=request_id, + ) # Extract images from result # Handle nested OmniRequestOutput structure where images might be in request_output images = getattr(result.request_output, "images", []) diff --git a/vllm_omni/inputs/data.py b/vllm_omni/inputs/data.py index ec291e43e..01de61b96 100644 --- a/vllm_omni/inputs/data.py +++ b/vllm_omni/inputs/data.py @@ -1,4 +1,11 @@ -from typing import Any +import copy +import pprint +from dataclasses import asdict, dataclass, field +from typing import Any, TypeAlias + +from vllm import PromptType, SamplingParams + +from vllm_omni.lora.request import LoRARequest try: from typing import NotRequired @@ -7,7 +14,25 @@ except ImportError: from typing_extensions import NotRequired import torch -from vllm.inputs.data import EmbedsPrompt, TokenInputs, TokensPrompt +from vllm.inputs.data import EmbedsPrompt, TextPrompt, TokenInputs, TokensPrompt + + +class OmniTextPrompt(TextPrompt): + """Text prompt with optional embeddings and additional information. + + Extends TextPrompt to support prompt embeddings and additional + information payloads for direct transfer between pipeline stages. + + Attributes: + prompt_embeds: Optional tensor containing prompt embeddings + additional_information: Optional dictionary containing additional + information (tensors or lists) to pass along with the prompt + """ + + negative_prompt: NotRequired[str] + prompt_embeds: NotRequired[torch.Tensor] + negative_prompt_embeds: NotRequired[torch.Tensor] + additional_information: NotRequired[dict[str, Any]] class OmniTokensPrompt(TokensPrompt): @@ -22,7 +47,9 @@ class OmniTokensPrompt(TokensPrompt): information (tensors or lists) to pass along with the prompt """ + negative_prompt: NotRequired[str] prompt_embeds: NotRequired[torch.Tensor] + negative_prompt_embeds: NotRequired[list[torch.Tensor] | None] """The embeddings of the prompt.""" # New: optional additional information dictionary @@ -44,7 +71,9 @@ class OmniTokenInputs(TokenInputs): """ # New: optional prompt embeddings aligned with token ids + negative_prompt: NotRequired[str] prompt_embeds: NotRequired[torch.Tensor] + negative_prompt_embeds: NotRequired[list[torch.Tensor] | None] # New: optional additional information dictionary # Values may be torch.Tensor or list @@ -65,12 +94,19 @@ class OmniEmbedsPrompt(EmbedsPrompt): # New: optional prompt embeddings aligned with token ids prompt_embeds: NotRequired[torch.Tensor] + negative_prompt_embeds: NotRequired[list[torch.Tensor] | None] # New: optional additional information dictionary # Values may be torch.Tensor or list additional_information: NotRequired[dict[str, Any]] +# Must ensure that all additional prompt types are inherited from vLLM prompt types +# Because TypedDict doesn't support isinstance and are dict. Cannot distinguish them in runtime. +# Inheritance ensure that there are only additional fields but not removing fields--safe to route to LLM.generate() +OmniPromptType: TypeAlias = PromptType | OmniTextPrompt | OmniTokensPrompt | OmniEmbedsPrompt + + def token_inputs_omni( prompt_token_ids: list[int], prompt: str | None = None, @@ -106,3 +142,140 @@ def token_inputs_omni( inputs["additional_information"] = additional_information return inputs + + +@dataclass +class OmniDiffusionSamplingParams: + """ + The collection of sampling parameters passed to diffusion pipelines. + + This dataclass contains all information needed during the diffusion pipeline + execution, allowing methods to update specific components without needing + to manage numerous individual parameters. + """ + + # Additional text-related parameters + max_sequence_length: int | None = None + prompt_template: dict[str, Any] | None = None + do_classifier_free_guidance: bool = False + + # Batch info + num_outputs_per_prompt: int = 1 + seed: int | None = None + generator: torch.Generator | list[torch.Generator] | None = None + + # layered info + layers: int = 4 + + # cfg info + cfg_normalize: bool = False + + # caption language + use_en_prompt: bool = False + + # different bucket in (640, 1024) to determine the condition and output resolution + resolution: int = 640 + + # Tracking if embeddings are already processed + is_prompt_processed: bool = False + + # Latent tensors + latents: torch.Tensor | None = None + raw_latent_shape: torch.Tensor | None = None + noise_pred: torch.Tensor | None = None + image_latent: torch.Tensor | None = None + + # Latent dimensions + height_latents: list[int] | int | None = None + width_latents: list[int] | int | None = None + num_frames: int = 1 # Default for image models + num_frames_round_down: bool = False # Whether to round down num_frames if it's not divisible by num_gpus + + # Original dimensions (before VAE scaling) + height: int | None = None + width: int | None = None + fps: int | None = None + height_not_provided: bool = False + width_not_provided: bool = False + + # Timesteps + timesteps: torch.Tensor | None = None + timestep: torch.Tensor | float | int | None = None + step_index: int | None = None + boundary_ratio: float | None = None + + # Scheduler parameters + num_inference_steps: int = 50 + guidance_scale: float = 0.0 + guidance_scale_provided: bool = False + guidance_scale_2: float | None = None + guidance_rescale: float = 0.0 + eta: float = 0.0 + sigmas: list[float] | None = None + + true_cfg_scale: float | None = None # qwen-image specific now + + n_tokens: int | None = None + extra_step_kwargs: dict[str, Any] = field(default_factory=dict) + + # [Omni] KV Cache Transfer, for bagel model now + past_key_values: Any | None = None # Injected KV Cache + kv_metadata: dict[str, Any] | None = None # Metadata for KV Cache (e.g., kv_lens, ropes) + need_kv_receive: bool = True # Flag to indicate if this request expects KV transfer + + # Component modules + modules: dict[str, Any] = field(default_factory=dict) + + return_trajectory_latents: bool = False + return_trajectory_decoded: bool = False + trajectory_timesteps: list[torch.Tensor] | None = None + trajectory_latents: torch.Tensor | None = None + + # Extra parameters that might be needed by specific pipeline implementations + extra_args: dict[str, Any] = field(default_factory=dict) + + # Misc + save_output: bool = True + return_frames: bool = False + + # LoRA + lora_request: LoRARequest | None = None + lora_scale: float = 1.0 + + # STA parameters + STA_param: list | None = None + is_cfg_negative: bool = False + mask_search_final_result_pos: list[list] | None = None + mask_search_final_result_neg: list[list] | None = None + + # VSA parameters + VSA_sparsity: float = 0.0 + # perf_logger: PerformanceLogger | None = None + + # stage logging + # logging_info: PipelineLoggingInfo = field(default_factory=PipelineLoggingInfo) + + # profile + profile: bool = False + num_profiled_timesteps: int = 8 + + # debugging + debug: bool = False + + # results + output: torch.Tensor | None = None + + @property + def batch_size(self): + # This class is changed to only represent a single prompt request + # Only adjust batch size for number of videos per prompt + return self.num_outputs_per_prompt + + def __str__(self): + return pprint.pformat(asdict(self), indent=2, width=120) + + def clone(self) -> "OmniDiffusionSamplingParams": + return copy.deepcopy(self) + + +OmniSamplingParams: TypeAlias = SamplingParams | OmniDiffusionSamplingParams diff --git a/vllm_omni/model_executor/stage_configs/bagel.yaml b/vllm_omni/model_executor/stage_configs/bagel.yaml index cf5dff50d..96bdc01b6 100644 --- a/vllm_omni/model_executor/stage_configs/bagel.yaml +++ b/vllm_omni/model_executor/stage_configs/bagel.yaml @@ -58,13 +58,7 @@ stage_args: final_output_type: image is_comprehension: false default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 seed: 52 - detokenize: True - repetition_penalty: 1.0 # Runtime edges runtime: diff --git a/vllm_omni/outputs.py b/vllm_omni/outputs.py index 958cebd3e..d67922000 100644 --- a/vllm_omni/outputs.py +++ b/vllm_omni/outputs.py @@ -6,6 +6,8 @@ from PIL import Image from vllm.outputs import RequestOutput from vllm.v1.outputs import ModelRunnerOutput +from vllm_omni.inputs.data import OmniPromptType + class OmniModelRunnerOutput(ModelRunnerOutput): """Model runner output for omni models. @@ -54,7 +56,7 @@ class OmniRequestOutput: # Diffusion model fields images: list[Image.Image] = field(default_factory=list) - prompt: str | None = None + prompt: OmniPromptType | None = None latents: torch.Tensor | None = None metrics: dict[str, Any] = field(default_factory=dict) multimodal_output: dict[str, Any] = field(default_factory=dict) @@ -89,7 +91,7 @@ class OmniRequestOutput: cls, request_id: str, images: list[Image.Image], - prompt: str | None = None, + prompt: OmniPromptType | None = None, metrics: dict[str, Any] | None = None, latents: torch.Tensor | None = None, multimodal_output: dict[str, Any] | None = None, From ae96aa8a8bede85bd0699e1108c1c38db170a7d7 Mon Sep 17 00:00:00 2001 From: JohnJan Date: Wed, 28 Jan 2026 16:28:57 +0800 Subject: [PATCH 39/83] [Model]: add FLUX.1-dev model (#853) --- docs/models/supported_models.md | 1 + .../diffusion/parallelism_acceleration.md | 1 + .../diffusion/cache/cache_dit_backend.py | 2 +- vllm_omni/diffusion/models/flux/__init__.py | 17 + .../diffusion/models/flux/flux_transformer.py | 638 ++++++++++++++++ .../diffusion/models/flux/pipeline_flux.py | 712 ++++++++++++++++++ vllm_omni/diffusion/registry.py | 6 + 7 files changed, 1376 insertions(+), 1 deletion(-) create mode 100644 vllm_omni/diffusion/models/flux/__init__.py create mode 100644 vllm_omni/diffusion/models/flux/flux_transformer.py create mode 100644 vllm_omni/diffusion/models/flux/pipeline_flux.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index e713d7934..269c9261e 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -33,6 +33,7 @@ th { |`LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | |`StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | |`Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | +|`FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | |`StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | |`Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | |`Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | diff --git a/docs/user_guide/diffusion/parallelism_acceleration.md b/docs/user_guide/diffusion/parallelism_acceleration.md index 45e53294e..1a9f540ad 100644 --- a/docs/user_guide/diffusion/parallelism_acceleration.md +++ b/docs/user_guide/diffusion/parallelism_acceleration.md @@ -30,6 +30,7 @@ The following table shows which models are currently supported by parallelism me | **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ✅ | ✅ | ❌ | ✅ (TP=2 only) | | **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ❌ | ❌ | ❌ | | **FLUX.2-klein** | `black-forest-labs/FLUX.2-klein-4B` | ❌ | ❌ | ❌ | ✅ | +| **FLUX.1-dev** | `black-forest-labs/FLUX.1-dev` | ❌ | ❌ | ❌ | ✅ | !!! note "TP Limitations for Diffusion Models" diff --git a/vllm_omni/diffusion/cache/cache_dit_backend.py b/vllm_omni/diffusion/cache/cache_dit_backend.py index 0c43659c9..ff2c98002 100644 --- a/vllm_omni/diffusion/cache/cache_dit_backend.py +++ b/vllm_omni/diffusion/cache/cache_dit_backend.py @@ -791,7 +791,7 @@ def enable_cache_for_bagel(pipeline: Any, cache_config: Any) -> Callable[[int], CUSTOM_DIT_ENABLERS.update( { "WanPipeline": enable_cache_for_wan22, - "FluxPipeline": enable_cache_for_flux, + # "FluxPipeline": enable_cache_for_flux, "LongCatImagePipeline": enable_cache_for_longcat_image, "LongCatImageEditPipeline": enable_cache_for_longcat_image, "StableDiffusion3Pipeline": enable_cache_for_sd3, diff --git a/vllm_omni/diffusion/models/flux/__init__.py b/vllm_omni/diffusion/models/flux/__init__.py new file mode 100644 index 000000000..cad3508a1 --- /dev/null +++ b/vllm_omni/diffusion/models/flux/__init__.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""FLUX.1-dev diffusion model components.""" + +from vllm_omni.diffusion.models.flux.flux_transformer import ( + FluxTransformer2DModel, +) +from vllm_omni.diffusion.models.flux.pipeline_flux import ( + FluxPipeline, + get_flux_post_process_func, +) + +__all__ = [ + "FluxPipeline", + "FluxTransformer2DModel", + "get_flux_post_process_func", +] diff --git a/vllm_omni/diffusion/models/flux/flux_transformer.py b/vllm_omni/diffusion/models/flux/flux_transformer.py new file mode 100644 index 000000000..faf6d08d3 --- /dev/null +++ b/vllm_omni/diffusion/models/flux/flux_transformer.py @@ -0,0 +1,638 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Iterable +from typing import Any + +import torch +import torch.nn.functional as F +from diffusers.models.embeddings import ( + CombinedTimestepGuidanceTextProjEmbeddings, + CombinedTimestepTextProjEmbeddings, + get_1d_rotary_pos_embed, +) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.normalization import AdaLayerNormContinuous, AdaLayerNormZero, AdaLayerNormZeroSingle +from diffusers.utils import is_torch_npu_available +from torch import nn +from vllm.distributed import get_tensor_model_parallel_world_size, tensor_model_parallel_all_gather +from vllm.logger import init_logger +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ColumnParallelLinear, QKVParallelLinear, RowParallelLinear +from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +from vllm_omni.diffusion.attention.layer import Attention +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.layers.rope import RotaryEmbedding + +logger = init_logger(__name__) + + +class ColumnParallelApproxGELU(nn.Module): + def __init__(self, dim_in: int, dim_out: int, *, approximate: str, bias: bool = True): + super().__init__() + self.proj = ColumnParallelLinear( + dim_in, + dim_out, + bias=bias, + gather_output=False, + return_bias=False, + ) + self.approximate = approximate + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj(x) + return F.gelu(x, approximate=self.approximate) + + +class FeedForward(nn.Module): + def __init__( + self, + dim: int, + dim_out: int | None = None, + mult: int = 4, + activation_fn: str = "gelu-approximate", + inner_dim: int | None = None, + bias: bool = True, + ) -> None: + super().__init__() + + assert activation_fn == "gelu-approximate", "Only gelu-approximate is supported." + + inner_dim = inner_dim or int(dim * mult) + dim_out = dim_out or dim + + layers: list[nn.Module] = [ + ColumnParallelApproxGELU(dim, inner_dim, approximate="tanh", bias=bias), + nn.Identity(), # placeholder for weight loading + RowParallelLinear( + inner_dim, + dim_out, + input_is_parallel=True, + return_bias=False, + ), + ] + + self.net = nn.ModuleList(layers) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + for module in self.net: + hidden_states = module(hidden_states) + return hidden_states + + +class FluxAttention(torch.nn.Module): + def __init__( + self, + query_dim: int, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + added_kv_proj_dim: int | None = None, + added_proj_bias: bool | None = True, + out_bias: bool = True, + eps: float = 1e-5, + out_dim: int = None, + context_pre_only: bool | None = None, + pre_only: bool = False, + ): + super().__init__() + + self.head_dim = dim_head + self.inner_dim = out_dim if out_dim is not None else dim_head * heads + self.query_dim = query_dim + self.use_bias = bias + self.dropout = dropout + self.out_dim = out_dim if out_dim is not None else query_dim + self.context_pre_only = context_pre_only + self.pre_only = pre_only + self.heads = out_dim // dim_head if out_dim is not None else heads + self.added_kv_proj_dim = added_kv_proj_dim + self.added_proj_bias = added_proj_bias + + self.norm_q = RMSNorm(dim_head, eps=eps) + self.norm_k = RMSNorm(dim_head, eps=eps) + + self.to_qkv = QKVParallelLinear( + hidden_size=query_dim, + head_size=self.head_dim, + total_num_heads=self.heads, + bias=bias, + ) + + if not self.pre_only: + self.to_out = nn.ModuleList( + [ + RowParallelLinear( + self.inner_dim, + self.out_dim, + bias=out_bias, + input_is_parallel=True, + return_bias=False, + ), + nn.Dropout(dropout), + ] + ) + + if added_kv_proj_dim is not None: + self.norm_added_q = RMSNorm(dim_head, eps=eps) + self.norm_added_k = RMSNorm(dim_head, eps=eps) + + self.add_kv_proj = QKVParallelLinear( + hidden_size=self.added_kv_proj_dim, + head_size=self.head_dim, + total_num_heads=self.heads, + bias=added_proj_bias, + ) + + self.to_add_out = RowParallelLinear( + self.inner_dim, + query_dim, + bias=out_bias, + input_is_parallel=True, + return_bias=False, + ) + + self.rope = RotaryEmbedding(is_neox_style=False) + self.attn = Attention( + num_heads=self.to_qkv.num_heads, + head_size=self.head_dim, + softmax_scale=1.0 / (self.head_dim**0.5), + causal=False, + num_kv_heads=self.to_qkv.num_kv_heads, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor | None = None, + image_rotary_emb: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + qkv, _ = self.to_qkv(hidden_states) + q_size = self.to_qkv.num_heads * self.head_dim + kv_size = self.to_qkv.num_kv_heads * self.head_dim + query, key, value = qkv.split([q_size, kv_size, kv_size], dim=-1) + + query = query.unflatten(-1, (self.to_qkv.num_heads, -1)) + key = key.unflatten(-1, (self.to_qkv.num_kv_heads, -1)) + value = value.unflatten(-1, (self.to_qkv.num_kv_heads, -1)) + + query = self.norm_q(query) + key = self.norm_k(key) + + if self.added_kv_proj_dim is not None: + encoder_qkv, _ = self.add_kv_proj(encoder_hidden_states) + add_q_size = self.add_kv_proj.num_heads * self.head_dim + add_kv_size = self.add_kv_proj.num_kv_heads * self.head_dim + encoder_query, encoder_key, encoder_value = encoder_qkv.split( + [add_q_size, add_kv_size, add_kv_size], dim=-1 + ) + + encoder_query = encoder_query.unflatten(-1, (self.add_kv_proj.num_heads, -1)) + encoder_key = encoder_key.unflatten(-1, (self.add_kv_proj.num_kv_heads, -1)) + encoder_value = encoder_value.unflatten(-1, (self.add_kv_proj.num_kv_heads, -1)) + + encoder_query = self.norm_added_q(encoder_query) + encoder_key = self.norm_added_k(encoder_key) + + query = torch.cat([encoder_query, query], dim=1) + key = torch.cat([encoder_key, key], dim=1) + value = torch.cat([encoder_value, value], dim=1) + + if image_rotary_emb is not None: + cos, sin = image_rotary_emb # [S, D/2] + cos = cos.to(query.dtype) + sin = sin.to(query.dtype) + query = self.rope(query, cos, sin) + key = self.rope(key, cos, sin) + + hidden_states = self.attn( + query, + key, + value, + ) + hidden_states = hidden_states.flatten(2, 3) + hidden_states = hidden_states.to(query.dtype) + + if encoder_hidden_states is not None: + encoder_hidden_states, hidden_states = hidden_states.split_with_sizes( + [encoder_hidden_states.shape[1], hidden_states.shape[1] - encoder_hidden_states.shape[1]], dim=1 + ) + hidden_states = self.to_out[0](hidden_states) + hidden_states = self.to_out[1](hidden_states) + encoder_hidden_states = self.to_add_out(encoder_hidden_states) + + return hidden_states, encoder_hidden_states + else: + # For single-stream blocks, there's no to_out (RowParallelLinear) to handle the reduction + if get_tensor_model_parallel_world_size() > 1: + hidden_states = tensor_model_parallel_all_gather(hidden_states, dim=-1) + return hidden_states + + +class FluxTransformerBlock(nn.Module): + def __init__( + self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6 + ): + super().__init__() + + self.norm1 = AdaLayerNormZero(dim) + self.norm1_context = AdaLayerNormZero(dim) + + self.attn = FluxAttention( + query_dim=dim, + added_kv_proj_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + context_pre_only=False, + bias=True, + eps=eps, + ) + + self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff = FeedForward(dim=dim, dim_out=dim) + + self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff_context = FeedForward(dim=dim, dim_out=dim) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + joint_attention_kwargs: dict[str, Any] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) + + norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( + encoder_hidden_states, emb=temb + ) + joint_attention_kwargs = joint_attention_kwargs or {} + + # Attention. + attention_outputs = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + image_rotary_emb=image_rotary_emb, + **joint_attention_kwargs, + ) + + if len(attention_outputs) == 2: + attn_output, context_attn_output = attention_outputs + elif len(attention_outputs) == 3: + attn_output, context_attn_output, ip_attn_output = attention_outputs + + # Process attention outputs for the `hidden_states`. + attn_output = gate_msa.unsqueeze(1) * attn_output + hidden_states = hidden_states + attn_output + + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + ff_output = self.ff(norm_hidden_states) + ff_output = gate_mlp.unsqueeze(1) * ff_output + + hidden_states = hidden_states + ff_output + if len(attention_outputs) == 3: + hidden_states = hidden_states + ip_attn_output + + # Process attention outputs for the `encoder_hidden_states`. + context_attn_output = c_gate_msa.unsqueeze(1) * context_attn_output + encoder_hidden_states = encoder_hidden_states + context_attn_output + + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] + + context_ff_output = self.ff_context(norm_encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class FluxSingleTransformerBlock(nn.Module): + def __init__(self, dim: int, num_attention_heads: int, attention_head_dim: int, mlp_ratio: float = 4.0): + super().__init__() + self.mlp_hidden_dim = int(dim * mlp_ratio) + + self.norm = AdaLayerNormZeroSingle(dim) + self.proj_mlp = nn.Linear(dim, self.mlp_hidden_dim) + self.act_mlp = nn.GELU(approximate="tanh") + self.proj_out = nn.Linear(dim + self.mlp_hidden_dim, dim) + + self.attn = FluxAttention( + query_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + bias=True, + eps=1e-6, + pre_only=True, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None, + joint_attention_kwargs: dict[str, Any] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + text_seq_len = encoder_hidden_states.shape[1] + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + residual = hidden_states + norm_hidden_states, gate = self.norm(hidden_states, emb=temb) + mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states)) + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + hidden_states=norm_hidden_states, + image_rotary_emb=image_rotary_emb, + **joint_attention_kwargs, + ) + + hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + gate = gate.unsqueeze(1) + hidden_states = gate * self.proj_out(hidden_states) + hidden_states = residual + hidden_states + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + encoder_hidden_states, hidden_states = hidden_states[:, :text_seq_len], hidden_states[:, text_seq_len:] + return encoder_hidden_states, hidden_states + + +class FluxPosEmbed(nn.Module): + # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 + def __init__(self, theta: int, axes_dim: list[int]): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + n_axes = ids.shape[-1] + cos_out = [] + sin_out = [] + pos = ids.float() + is_mps = ids.device.type == "mps" + is_npu = ids.device.type == "npu" + freqs_dtype = torch.float32 if (is_mps or is_npu) else torch.float64 + for i in range(n_axes): + freqs_cis = get_1d_rotary_pos_embed( + self.axes_dim[i], + pos[:, i], + theta=self.theta, + use_real=False, + freqs_dtype=freqs_dtype, + ) + cos_out.append(freqs_cis.real) + sin_out.append(freqs_cis.imag) + freqs_cos = torch.cat(cos_out, dim=-1).to(ids.device) + freqs_sin = torch.cat(sin_out, dim=-1).to(ids.device) + return freqs_cos, freqs_sin + + +class FluxTransformer2DModel(nn.Module): + """ + The Transformer model introduced in Flux. + + Args: + od_config (`OmniDiffusionConfig`): + The configuration for the model. + patch_size (`int`, defaults to `1`): + Patch size to turn the input data into small patches. + in_channels (`int`, defaults to `64`): + The number of channels in the input. + out_channels (`int`, *optional*, defaults to `None`): + The number of channels in the output. If not specified, it defaults to `in_channels`. + num_layers (`int`, defaults to `19`): + The number of layers of dual stream DiT blocks to use. + num_single_layers (`int`, defaults to `38`): + The number of layers of single stream DiT blocks to use. + attention_head_dim (`int`, defaults to `128`): + The number of dimensions to use for each attention head. + num_attention_heads (`int`, defaults to `24`): + The number of attention heads to use. + joint_attention_dim (`int`, defaults to `4096`): + The number of dimensions to use for the joint attention (embedding/channel dimension of + `encoder_hidden_states`). + pooled_projection_dim (`int`, defaults to `768`): + The number of dimensions to use for the pooled projection. + guidance_embeds (`bool`, defaults to `False`): + Whether to use guidance embeddings for guidance-distilled variant of the model. + axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`): + The dimensions to use for the rotary positional embeddings. + """ + + # the small and frequently-repeated block(s) of a model + # -- typically a transformer layer + # used for torch compile optimizations + _repeated_blocks = ["FluxTransformerBlock"] + + def __init__( + self, + od_config: OmniDiffusionConfig, + patch_size: int = 1, + in_channels: int = 64, + out_channels: int = None, + num_layers: int = 19, + num_single_layers: int = 38, + attention_head_dim: int = 128, + num_attention_heads: int = 24, + joint_attention_dim: int = 4096, + pooled_projection_dim: int = 768, + guidance_embeds: bool = True, + axes_dims_rope: tuple[int, int, int] = (16, 56, 56), + ): + super().__init__() + model_config = od_config.tf_model_config + num_layers = model_config.num_layers + self.parallel_config = od_config.parallel_config + self.in_channels = in_channels + self.out_channels = out_channels or in_channels + self.inner_dim = num_attention_heads * attention_head_dim + self.guidance_embeds = guidance_embeds + + self.pos_embed = FluxPosEmbed(theta=10000, axes_dim=axes_dims_rope) + text_time_guidance_cls = ( + CombinedTimestepGuidanceTextProjEmbeddings if guidance_embeds else CombinedTimestepTextProjEmbeddings + ) + self.time_text_embed = text_time_guidance_cls( + embedding_dim=self.inner_dim, pooled_projection_dim=pooled_projection_dim + ) + + self.context_embedder = nn.Linear(joint_attention_dim, self.inner_dim) + self.x_embedder = nn.Linear(in_channels, self.inner_dim) + + self.transformer_blocks = nn.ModuleList( + [ + FluxTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + ) + for _ in range(num_layers) + ] + ) + + self.single_transformer_blocks = nn.ModuleList( + [ + FluxSingleTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + ) + for _ in range(num_single_layers) + ] + ) + + self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + pooled_projections: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor | None = None, + joint_attention_kwargs: dict[str, Any] | None = None, + return_dict: bool = True, + ) -> torch.Tensor | Transformer2DModelOutput: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + pooled_projections (`torch.Tensor` of shape `(batch_size, projection_dim)`): Embeddings projected + from the embeddings of input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + img_ids: (`torch.Tensor`): + The position ids for image tokens. + txt_ids (`torch.Tensor`): + The position ids for text tokens. + guidance (`torch.Tensor`): + Guidance embeddings for guidance-distilled variant of the model. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + + hidden_states = self.x_embedder(hidden_states) + timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype) * 1000 + + if guidance is not None: + guidance = guidance.to(device=hidden_states.device, dtype=hidden_states.dtype) * 1000 + + temb = ( + self.time_text_embed(timestep, pooled_projections) + if guidance is None + else self.time_text_embed(timestep, guidance, pooled_projections) + ) + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + + if txt_ids.ndim == 3: + logger.warning( + "Passing `txt_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + txt_ids = txt_ids[0] + if img_ids.ndim == 3: + logger.warning( + "Passing `img_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + img_ids = img_ids[0] + + ids = torch.cat((txt_ids, img_ids), dim=0) + if is_torch_npu_available(): + freqs_cos, freqs_sin = self.pos_embed(ids.cpu()) + image_rotary_emb = (freqs_cos.npu(), freqs_sin.npu()) + else: + image_rotary_emb = self.pos_embed(ids) + + for index_block, block in enumerate(self.transformer_blocks): + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + for index_block, block in enumerate(self.single_transformer_blocks): + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + # self-attn + (".to_qkv", ".to_q", "q"), + (".to_qkv", ".to_k", "k"), + (".to_qkv", ".to_v", "v"), + # cross-attn + (".add_kv_proj", ".add_q_proj", "q"), + (".add_kv_proj", ".add_k_proj", "k"), + (".add_kv_proj", ".add_v_proj", "v"), + ] + + params_dict = dict(self.named_parameters()) + + # we need to load the buffers for beta and eps (XIELU) + for name, buffer in self.named_buffers(): + if name.endswith(".beta") or name.endswith(".eps"): + params_dict[name] = buffer + + loaded_params: set[str] = set() + for name, loaded_weight in weights: + original_name = name + lookup_name = name + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in original_name: + continue + lookup_name = original_name.replace(weight_name, param_name) + param = params_dict[lookup_name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + if lookup_name not in params_dict and ".to_out.0." in lookup_name: + lookup_name = lookup_name.replace(".to_out.0.", ".to_out.") + param = params_dict[lookup_name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(original_name) + loaded_params.add(lookup_name) + return loaded_params diff --git a/vllm_omni/diffusion/models/flux/pipeline_flux.py b/vllm_omni/diffusion/models/flux/pipeline_flux.py new file mode 100644 index 000000000..01e0bfba9 --- /dev/null +++ b/vllm_omni/diffusion/models/flux/pipeline_flux.py @@ -0,0 +1,712 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import inspect +import json +import logging +import os +from collections.abc import Iterable +from typing import Any + +import numpy as np +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import TextualInversionLoaderMixin +from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL +from diffusers.schedulers.scheduling_flow_match_euler_discrete import ( + FlowMatchEulerDiscreteScheduler, +) +from diffusers.utils.torch_utils import randn_tensor +from torch import nn +from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5TokenizerFast +from vllm.model_executor.models.utils import AutoWeightsLoader + +from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.utils import get_local_device +from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.models.flux import FluxTransformer2DModel +from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific + +logger = logging.getLogger(__name__) + + +def get_flux_post_process_func( + od_config: OmniDiffusionConfig, +): + if od_config.output_type == "latent": + return lambda x: x + model_name = od_config.model + if os.path.exists(model_name): + model_path = model_name + else: + model_path = download_weights_from_hf_specific(model_name, None, ["*"]) + + vae_config_path = os.path.join(model_path, "vae/config.json") + with open(vae_config_path) as f: + vae_config = json.load(f) + vae_scale_factor = 2 ** (len(vae_config["block_out_channels"]) - 1) if "block_out_channels" in vae_config else 8 + + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + + def post_process_func(images: torch.Tensor): + return image_processor.postprocess(images) + + return post_process_func + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: int | None = None, + device: str | torch.device | None = None, + timesteps: list[int] | None = None, + sigmas: list[float] | None = None, + **kwargs, +) -> tuple[torch.Tensor, int]: + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class FluxPipeline( + nn.Module, +): + def __init__( + self, + *, + od_config: OmniDiffusionConfig, + prefix: str = "", + ): + super().__init__() + self.od_config = od_config + self.weights_sources = [ + DiffusersPipelineLoader.ComponentSource( + model_or_path=od_config.model, + subfolder="transformer", + revision=None, + prefix="transformer.", + fall_back_to_pt=True, + ) + ] + + self.device = get_local_device() + model = od_config.model + # Check if model is a local path + local_files_only = os.path.exists(model) + + self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + model, subfolder="scheduler", local_files_only=local_files_only + ) + self.text_encoder = CLIPTextModel.from_pretrained( + model, subfolder="text_encoder", local_files_only=local_files_only + ) + self.text_encoder_2 = T5EncoderModel.from_pretrained( + model, subfolder="text_encoder_2", local_files_only=local_files_only + ) + self.vae = AutoencoderKL.from_pretrained(model, subfolder="vae", local_files_only=local_files_only).to( + self.device + ) + self.transformer = FluxTransformer2DModel(od_config=od_config) + + self.tokenizer = CLIPTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only) + self.tokenizer_2 = T5TokenizerFast.from_pretrained( + model, subfolder="tokenizer_2", local_files_only=local_files_only + ) + + self.stage = None + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 + # Flux latents are turned into 2x2 patches and packed. + # This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + # self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.tokenizer_max_length = ( + self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77 + ) + self.default_sample_size = 128 + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + max_sequence_length=None, + ): + if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0: + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} " + f"but are {height} and {width}. Dimensions will be resized accordingly" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. " + "Make sure to generate `pooled_prompt_embeds` from the same text encoder " + "that was used to generate `prompt_embeds`." + ) + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. " + "Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder " + "that was used to generate `negative_prompt_embeds`." + ) + + if max_sequence_length is not None and max_sequence_length > 512: + raise ValueError(f"`max_sequence_length` cannot be greater than 512 but is {max_sequence_length}") + + def _get_t5_prompt_embeds( + self, + prompt: str | list[str] = None, + num_images_per_prompt: int = 1, + max_sequence_length: int = 512, + dtype: torch.dtype | None = None, + ): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer_2) + + text_inputs = self.tokenizer_2( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_length=False, + return_overflowing_tokens=False, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder_2(text_input_ids.to(self.device), output_hidden_states=False)[0] + + dtype = self.text_encoder_2.dtype + prompt_embeds = prompt_embeds.to(dtype=dtype, device=self.device) + + _, seq_len, _ = prompt_embeds.shape + + # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + return prompt_embeds + + def _get_clip_prompt_embeds( + self, + prompt: str | list[str], + num_images_per_prompt: int = 1, + ): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer_max_length, + truncation=True, + return_overflowing_tokens=False, + return_length=False, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer_max_length} tokens: {removed_text}" + ) + prompt_embeds = self.text_encoder(text_input_ids.to(self.device), output_hidden_states=False) + + # Use pooled output of CLIPTextModel + prompt_embeds = prompt_embeds.pooler_output + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=self.device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, -1) + + return prompt_embeds + + def encode_prompt( + self, + prompt: str | list[str], + prompt_2: str | list[str], + num_images_per_prompt: int = 1, + prompt_embeds: torch.FloatTensor | None = None, + pooled_prompt_embeds: torch.FloatTensor | None = None, + max_sequence_length: int = 512, + ): + r""" + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in all text-encoders + num_images_per_prompt (`int`): + number of images that should be generated per prompt + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + """ + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # We only use the pooled prompt output from the CLIPTextModel + pooled_prompt_embeds = self._get_clip_prompt_embeds( + prompt=prompt, + num_images_per_prompt=num_images_per_prompt, + ) + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt_2, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + + dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype + text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=self.device, dtype=dtype) + + return prompt_embeds, pooled_prompt_embeds, text_ids + + @staticmethod + def _prepare_latent_image_ids(batch_size, height, width, device, dtype): + latent_image_ids = torch.zeros(height, width, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids.to(device=device, dtype=dtype) + + @staticmethod + def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + @staticmethod + def _unpack_latents(latents, height, width, vae_scale_factor): + batch_size, num_patches, channels = latents.shape + + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), height, width) + + return latents + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, num_channels_latents, height, width) + + if latents is not None: + latent_image_ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype) + return latents.to(device=device, dtype=dtype), latent_image_ids + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width) + + latent_image_ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype) + + return latents, latent_image_ids + + def prepare_timesteps(self, num_inference_steps, sigmas, image_seq_len): + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + sigmas=sigmas, + mu=mu, + ) + return timesteps, num_inference_steps + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def joint_attention_kwargs(self): + return self._joint_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + def diffuse( + self, + prompt_embeds, + pooled_prompt_embeds, + negative_prompt_embeds, + negative_pooled_prompt_embeds, + latents, + latent_image_ids, + text_ids, + negative_text_ids, + timesteps, + do_true_cfg, + guidance, + true_cfg_scale, + ): + """Diffusion loop with optional image conditioning.""" + self.scheduler.set_begin_index(0) + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + # broadcast to batch dimension and place on same device/dtype as latents + timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) + + self.transformer.do_true_cfg = do_true_cfg # used in teacache hook + # Forward pass for positive prompt (or unconditional if no CFG) + noise_pred = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + guidance=guidance, + pooled_projections=pooled_prompt_embeds, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + joint_attention_kwargs=self.joint_attention_kwargs, + return_dict=False, + )[0] + + # Forward pass for negative prompt (CFG) + if do_true_cfg: + neg_noise_pred = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + guidance=guidance, + pooled_projections=negative_pooled_prompt_embeds, + encoder_hidden_states=negative_prompt_embeds, + txt_ids=negative_text_ids, + img_ids=latent_image_ids, + joint_attention_kwargs=self.joint_attention_kwargs, + return_dict=False, + )[0] + noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + return latents + + def forward( + self, + req: OmniDiffusionRequest, + prompt: str | list[str] = None, + prompt_2: str | list[str] = None, + negative_prompt: str | list[str] = None, + negative_prompt_2: str | list[str] = None, + true_cfg_scale: float = 1.0, + height: int | None = None, + width: int | None = None, + num_inference_steps: int = 28, + sigmas: list[float] | None = None, + guidance_scale: float = 3.5, + num_images_per_prompt: int = 1, + generator: torch.Generator | list[torch.Generator] | None = None, + latents: torch.FloatTensor | None = None, + prompt_embeds: torch.FloatTensor | None = None, + pooled_prompt_embeds: torch.FloatTensor | None = None, + negative_prompt_embeds: torch.FloatTensor | None = None, + negative_pooled_prompt_embeds: torch.FloatTensor | None = None, + output_type: str | None = "pil", + return_dict: bool = True, + joint_attention_kwargs: dict[str, Any] | None = None, + callback_on_step_end_tensor_inputs: list[str] = ["latents"], + max_sequence_length: int = 512, + ): + """Forward pass for flux.""" + prompt = req.prompt if req.prompt is not None else prompt + negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt + height = req.height or self.default_sample_size * self.vae_scale_factor + width = req.width or self.default_sample_size * self.vae_scale_factor + sigmas = req.sigmas or sigmas + num_inference_steps = req.num_inference_steps or num_inference_steps + generator = req.generator or generator + req_num_outputs = getattr(req, "num_outputs_per_prompt", None) + if req_num_outputs and req_num_outputs > 0: + num_images_per_prompt = req_num_outputs + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + self._guidance_scale = guidance_scale + self._joint_attention_kwargs = joint_attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None + ) + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + + ( + prompt_embeds, + pooled_prompt_embeds, + text_ids, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + + negative_text_ids = None + if do_true_cfg: + ( + negative_prompt_embeds, + negative_pooled_prompt_embeds, + negative_text_ids, + ) = self.encode_prompt( + prompt=negative_prompt, + prompt_2=negative_prompt_2, + prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=negative_pooled_prompt_embeds, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + + # 4. Prepare latent variables + num_channels_latents = self.transformer.in_channels // 4 + latents, latent_image_ids = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + self.device, + generator, + latents, + ) + + # 5. Prepare timesteps + timesteps, num_inference_steps = self.prepare_timesteps(num_inference_steps, sigmas, latents.shape[1]) + self._num_timesteps = len(timesteps) + + # handle guidance + if self.transformer.guidance_embeds: + guidance = torch.full([1], guidance_scale, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + else: + guidance = None + + if self.joint_attention_kwargs is None: + self._joint_attention_kwargs = {} + + latents = self.diffuse( + prompt_embeds, + pooled_prompt_embeds, + negative_prompt_embeds, + negative_pooled_prompt_embeds, + latents, + latent_image_ids, + text_ids, + negative_text_ids, + timesteps, + do_true_cfg, + guidance, + true_cfg_scale, + ) + + self._current_timestep = None + + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) + latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor + image = self.vae.decode(latents, return_dict=False)[0] + + return DiffusionOutput(output=image) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) diff --git a/vllm_omni/diffusion/registry.py b/vllm_omni/diffusion/registry.py index d4429aff6..024352479 100644 --- a/vllm_omni/diffusion/registry.py +++ b/vllm_omni/diffusion/registry.py @@ -90,6 +90,11 @@ _DIFFUSION_MODELS = { "pipeline_flux2_klein", "Flux2KleinPipeline", ), + "FluxPipeline": ( + "flux", + "pipeline_flux", + "FluxPipeline", + ), } @@ -227,6 +232,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = { "LongCatImageEditPipeline": "get_longcat_image_post_process_func", "StableDiffusion3Pipeline": "get_sd3_image_post_process_func", "Flux2KleinPipeline": "get_flux2_klein_post_process_func", + "FluxPipeline": "get_flux_post_process_func", } _DIFFUSION_PRE_PROCESS_FUNCS = { From 6fb405856691a1adcc52abf6811d51486894716b Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 28 Jan 2026 17:10:05 +0800 Subject: [PATCH 40/83] [BugFix] ignore mm data from stages to async omni (#954) Signed-off-by: dengyunyang <584797741@qq.com> --- vllm_omni/entrypoints/omni_stage.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/vllm_omni/entrypoints/omni_stage.py b/vllm_omni/entrypoints/omni_stage.py index 0a5ee55be..6bf1fc668 100644 --- a/vllm_omni/entrypoints/omni_stage.py +++ b/vllm_omni/entrypoints/omni_stage.py @@ -23,6 +23,7 @@ from vllm import PromptType, RequestOutput from vllm.inputs import TextPrompt from vllm.inputs.preprocess import InputPreprocessor from vllm.logger import init_logger +from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.tokenizers import TokenizerLike from vllm.usage.usage_lib import UsageContext @@ -1423,7 +1424,7 @@ async def _stage_worker_async( batch_request_ids, batch_request_outputs, _gen_ms_list, batch_metrics ): try: - r_outputs = [output] + r_outputs = [output_strip(output, omni_stage)] use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes) if use_shm: out_q.put( @@ -1509,3 +1510,23 @@ def make_stage_stats(_agg_total_tokens: int, _agg_total_gen_time_ms: float): from vllm_omni.entrypoints.log_utils import StageStats return StageStats(total_token=_agg_total_tokens, total_gen_time=_agg_total_gen_time_ms) + + +def output_strip(r_output: RequestOutput, omni_stage: OmniStage): + if omni_stage.final_output and omni_stage.final_output_type != "text": + return r_output + + if getattr(r_output, "finished", False): + return r_output + + mm_output = getattr(r_output, "multimodal_output", None) + if mm_output is not None: + r_output.multimodal_output = {} + + outputs = getattr(r_output, "outputs", None) + if outputs is not None: + for out in outputs: + if getattr(out, "multimodal_output", None): + out.multimodal_output = {} + + return r_output From 91a3d465eeb890c88c5710c90b43a54f9469d365 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Wed, 28 Jan 2026 17:41:28 +0800 Subject: [PATCH 41/83] Revert "[BugFix] ignore mm data from stages to async omni" (#1023) --- vllm_omni/entrypoints/omni_stage.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/vllm_omni/entrypoints/omni_stage.py b/vllm_omni/entrypoints/omni_stage.py index 6bf1fc668..0a5ee55be 100644 --- a/vllm_omni/entrypoints/omni_stage.py +++ b/vllm_omni/entrypoints/omni_stage.py @@ -23,7 +23,6 @@ from vllm import PromptType, RequestOutput from vllm.inputs import TextPrompt from vllm.inputs.preprocess import InputPreprocessor from vllm.logger import init_logger -from vllm.outputs import RequestOutput from vllm.sampling_params import SamplingParams from vllm.tokenizers import TokenizerLike from vllm.usage.usage_lib import UsageContext @@ -1424,7 +1423,7 @@ async def _stage_worker_async( batch_request_ids, batch_request_outputs, _gen_ms_list, batch_metrics ): try: - r_outputs = [output_strip(output, omni_stage)] + r_outputs = [output] use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes) if use_shm: out_q.put( @@ -1510,23 +1509,3 @@ def make_stage_stats(_agg_total_tokens: int, _agg_total_gen_time_ms: float): from vllm_omni.entrypoints.log_utils import StageStats return StageStats(total_token=_agg_total_tokens, total_gen_time=_agg_total_gen_time_ms) - - -def output_strip(r_output: RequestOutput, omni_stage: OmniStage): - if omni_stage.final_output and omni_stage.final_output_type != "text": - return r_output - - if getattr(r_output, "finished", False): - return r_output - - mm_output = getattr(r_output, "multimodal_output", None) - if mm_output is not None: - r_output.multimodal_output = {} - - outputs = getattr(r_output, "outputs", None) - if outputs is not None: - for out in outputs: - if getattr(out, "multimodal_output", None): - out.multimodal_output = {} - - return r_output From cb489682943153b5b6103d436474a8e7a9e28ca2 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Wed, 28 Jan 2026 18:42:39 +0800 Subject: [PATCH 42/83] [Bugfix] Modify output to model_runner_output (#1026) Signed-off-by: gcanlin --- vllm_omni/worker/npu/npu_ar_model_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm_omni/worker/npu/npu_ar_model_runner.py b/vllm_omni/worker/npu/npu_ar_model_runner.py index 0eedb61fb..1bb53b662 100644 --- a/vllm_omni/worker/npu/npu_ar_model_runner.py +++ b/vllm_omni/worker/npu/npu_ar_model_runner.py @@ -438,7 +438,7 @@ class NPUARModelRunner(OmniNPUModelRunner): pooler_output=(pooler_output if self.vllm_config.model_config.engine_output_type != "text" else None), kv_connector_output=kv_connector_output, ) - output.kv_extracted_req_ids = kv_extracted_req_ids + model_runner_output.kv_extracted_req_ids = kv_extracted_req_ids # -------------------------------------- Omni-new ------------------------------------------------- durations = ProfileExecuteDuration().pop_captured_sync() From c0beebbfe305c94556e9e620809e4aa942d2870d Mon Sep 17 00:00:00 2001 From: Samit <285365963@qq.com> Date: Wed, 28 Jan 2026 20:07:50 +0800 Subject: [PATCH 43/83] [Feature] Support cache-dit for Wan 2.2 inference (#1021) Signed-off-by: samithuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> --- .../text_to_image/text_to_image.py | 6 + .../text_to_video/text_to_video.py | 47 +++++++- .../diffusion/cache/cache_dit_backend.py | 113 ++++++++++-------- vllm_omni/diffusion/data.py | 1 + .../worker/gpu_diffusion_model_runner.py | 11 +- vllm_omni/diffusion/worker/npu/npu_worker.py | 6 +- vllm_omni/entrypoints/async_omni.py | 1 + vllm_omni/entrypoints/cli/serve.py | 5 + 8 files changed, 131 insertions(+), 59 deletions(-) diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 7fc6ec832..efc5010e0 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -73,6 +73,11 @@ def parse_args() -> argparse.Namespace: "Default: None (no cache acceleration)." ), ) + parser.add_argument( + "--enable-cache-dit-summary", + action="store_true", + help="Enable cache-dit summary logging after diffusion forward passes.", + ) parser.add_argument( "--ulysses_degree", type=int, @@ -166,6 +171,7 @@ def main(): vae_use_tiling=vae_use_tiling, cache_backend=args.cache_backend, cache_config=cache_config, + enable_cache_dit_summary=args.enable_cache_dit_summary, parallel_config=parallel_config, enforce_eager=args.enforce_eager, enable_cpu_offload=args.enable_cpu_offload, diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 054db95b3..49522278f 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -36,8 +36,29 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--flow_shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)." ) + parser.add_argument( + "--cache_backend", + type=str, + default=None, + choices=["cache_dit"], + help=( + "Cache backend to use for acceleration. " + "Options: 'cache_dit' (DBCache + SCM + TaylorSeer). " + "Default: None (no cache acceleration)." + ), + ) + parser.add_argument( + "--enable-cache-dit-summary", + action="store_true", + help="Enable cache-dit summary logging after diffusion forward passes.", + ) parser.add_argument("--output", type=str, default="wan22_output.mp4", help="Path to save the video (mp4).") parser.add_argument("--fps", type=int, default=24, help="Frames per second for the output video.") + parser.add_argument( + "--enforce_eager", + action="store_true", + help="Disable torch.compile and force eager execution.", + ) parser.add_argument( "--enable-cpu-offload", action="store_true", @@ -55,11 +76,6 @@ def parse_args() -> argparse.Namespace: default=1, help="Number of GPUs used for ring sequence parallelism.", ) - parser.add_argument( - "--enforce_eager", - action="store_true", - help="Disable torch.compile and force eager execution.", - ) return parser.parse_args() @@ -72,6 +88,24 @@ def main(): vae_use_slicing = is_npu() vae_use_tiling = is_npu() + # Wan2.2 cache-dit tuning (from cache-dit examples and cache_alignment). + cache_config = None + if args.cache_backend == "cache_dit": + cache_config = { + # DBCache parameters [cache-dit only] + "Fn_compute_blocks": 1, # Optimized for single-transformer models + "Bn_compute_blocks": 0, # Number of backward compute blocks + "max_warmup_steps": 4, # Maximum warmup steps (works for few-step models) + "max_cached_steps": 20, + "residual_diff_threshold": 0.24, # Higher threshold for more aggressive caching + "max_continuous_cached_steps": 3, # Limit to prevent precision degradation + # TaylorSeer parameters [cache-dit only] + "enable_taylorseer": False, # Disabled by default (not suitable for few-step models) + "taylorseer_order": 1, # TaylorSeer polynomial order + # SCM (Step Computation Masking) parameters [cache-dit only] + "scm_steps_mask_policy": None, # SCM mask policy: None (disabled), "slow", "medium", "fast", "ultra" + "scm_steps_policy": "dynamic", # SCM steps policy: "dynamic" or "static" + } # Configure parallel settings (only SP is supported for Wan) # Note: cfg_parallel and tensor_parallel are not implemented for Wan models parallel_config = DiffusionParallelConfig( @@ -88,6 +122,9 @@ def main(): vae_use_tiling=vae_use_tiling, boundary_ratio=args.boundary_ratio, flow_shift=args.flow_shift, + cache_backend=args.cache_backend, + cache_config=cache_config, + enable_cache_dit_summary=args.enable_cache_dit_summary, enable_cpu_offload=args.enable_cpu_offload, parallel_config=parallel_config, enforce_eager=args.enforce_eager, diff --git a/vllm_omni/diffusion/cache/cache_dit_backend.py b/vllm_omni/diffusion/cache/cache_dit_backend.py index ff2c98002..c922121e9 100644 --- a/vllm_omni/diffusion/cache/cache_dit_backend.py +++ b/vllm_omni/diffusion/cache/cache_dit_backend.py @@ -28,6 +28,13 @@ from vllm_omni.diffusion.data import DiffusionCacheConfig, OmniDiffusionConfig logger = init_logger(__name__) +# Small helper to centralize cache-dit summaries. +def cache_summary(pipeline: Any, details: bool = True) -> None: + cache_dit.summary(pipeline.transformer, details=details) + if hasattr(pipeline, "transformer_2"): + cache_dit.summary(pipeline.transformer_2, details=details) + + # Registry of custom cache-dit enablers for specific models # Maps pipeline names to their cache-dit enablement functions # Models in this registry require custom handling (e.g., dual-transformer architectures) @@ -71,53 +78,46 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int], A refresh function that can be called to update cache context with new num_inference_steps. """ - # Build DBCacheConfig for primary transformer - primary_cache_config = _build_db_cache_config(cache_config) - - # FIXME: secondary cache shares the same config with primary cache for now, - # but we should support different config for secondary transformer in the future - # Build DBCacheConfig for secondary transformer (can use same or different config) - secondary_cache_config = _build_db_cache_config(cache_config) - - # Build calibrator configs if TaylorSeer is enabled - primary_calibrator = None - secondary_calibrator = None - if cache_config.enable_taylorseer: - taylorseer_order = cache_config.taylorseer_order - primary_calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order) - secondary_calibrator = TaylorSeerCalibratorConfig(taylorseer_order=taylorseer_order) - logger.info(f"TaylorSeer enabled with order={taylorseer_order}") - - # Build ParamsModifier for each transformer - primary_modifier = ParamsModifier( - cache_config=primary_cache_config, - calibrator_config=primary_calibrator, - ) - secondary_modifier = ParamsModifier( - cache_config=secondary_cache_config, - calibrator_config=secondary_calibrator, - ) - - logger.info( - "Enabling cache-dit on Wan2.2 dual transformers with BlockAdapter: " - f"Fn={primary_cache_config.Fn_compute_blocks}, " - f"Bn={primary_cache_config.Bn_compute_blocks}, " - f"W={primary_cache_config.max_warmup_steps}, " - ) - - transformer = pipeline.transformer - transformer_2 = pipeline.transformer_2 - transformer_blocks = transformer.blocks - transformer_2_blocks = transformer_2.blocks - # Enable cache-dit using BlockAdapter for both transformers simultaneously cache_dit.enable_cache( BlockAdapter( - transformer=[transformer, transformer_2], - blocks=[transformer_blocks, transformer_2_blocks], - forward_pattern=[ForwardPattern.Pattern_2, ForwardPattern.Pattern_2], - params_modifiers=[primary_modifier, secondary_modifier], + transformer=[ + pipeline.transformer, + pipeline.transformer_2, + ], + blocks=[ + pipeline.transformer.blocks, + pipeline.transformer_2.blocks, + ], + forward_pattern=[ + ForwardPattern.Pattern_2, + ForwardPattern.Pattern_2, + ], + params_modifiers=[ + # high-noise transformer only have 30% steps + ParamsModifier( + cache_config=DBCacheConfig().reset( + max_warmup_steps=cache_config.max_warmup_steps, + max_cached_steps=cache_config.max_cached_steps, + ), + ), + ParamsModifier( + cache_config=DBCacheConfig().reset( + max_warmup_steps=2, + max_cached_steps=20, + ), + ), + ], has_separate_cfg=True, ), + cache_config=DBCacheConfig( + Fn_compute_blocks=cache_config.Fn_compute_blocks, + Bn_compute_blocks=cache_config.Bn_compute_blocks, + max_warmup_steps=cache_config.max_warmup_steps, + max_cached_steps=cache_config.max_cached_steps, + max_continuous_cached_steps=cache_config.max_continuous_cached_steps, + residual_diff_threshold=cache_config.residual_diff_threshold, + num_inference_steps=None, + ), ) # from https://github.com/vipshop/cache-dit/pull/542 @@ -133,8 +133,8 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int], Returns: A tuple of (num_high_noise_steps, num_low_noise_steps). """ - if pipeline.config.boundary_ratio is not None: - boundary_timestep = pipeline.config.boundary_ratio * pipeline.scheduler.config.num_train_timesteps + if pipeline.boundary_ratio is not None: + boundary_timestep = pipeline.boundary_ratio * pipeline.scheduler.config.num_train_timesteps else: boundary_timestep = None @@ -158,10 +158,21 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int], pipeline: The Wan2.2 pipeline instance. num_inference_steps: New number of inference steps. """ + num_high_noise_steps, num_low_noise_steps = _split_inference_steps(num_inference_steps) # Refresh context for high-noise transformer if cache_config.scm_steps_mask_policy is None: - cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_high_noise_steps, verbose=verbose) + # cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_high_noise_steps, verbose=verbose) + cache_dit.refresh_context( + pipeline.transformer, + num_inference_steps=num_high_noise_steps, + verbose=verbose, + ) + cache_dit.refresh_context( + pipeline.transformer_2, + num_inference_steps=num_low_noise_steps, + verbose=verbose, + ) else: cache_dit.refresh_context( pipeline.transformer, @@ -175,10 +186,6 @@ def enable_cache_for_wan22(pipeline: Any, cache_config: Any) -> Callable[[int], verbose=verbose, ) - # Refresh context for low-noise transformer - if cache_config.scm_steps_mask_policy is None: - cache_dit.refresh_context(pipeline.transformer_2, num_inference_steps=num_low_noise_steps, verbose=verbose) - else: cache_dit.refresh_context( pipeline.transformer_2, cache_config=DBCacheConfig().reset( @@ -790,8 +797,10 @@ def enable_cache_for_bagel(pipeline: Any, cache_config: Any) -> Callable[[int], # Register custom cache-dit enablers after function definitions CUSTOM_DIT_ENABLERS.update( { - "WanPipeline": enable_cache_for_wan22, - # "FluxPipeline": enable_cache_for_flux, + "Wan22Pipeline": enable_cache_for_wan22, + "Wan22I2VPipeline": enable_cache_for_wan22, + "Wan22TI2VPipeline": enable_cache_for_wan22, + "FluxPipeline": enable_cache_for_flux, "LongCatImagePipeline": enable_cache_for_longcat_image, "LongCatImageEditPipeline": enable_cache_for_longcat_image, "StableDiffusion3Pipeline": enable_cache_for_sd3, diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 1a98462aa..286ab9153 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -258,6 +258,7 @@ class OmniDiffusionConfig: # Cache backend configuration (NEW) cache_backend: str = "none" # "tea_cache", "deep_cache", etc. cache_config: DiffusionCacheConfig | dict[str, Any] = field(default_factory=dict) + enable_cache_dit_summary: bool = False # Distributed executor backend distributed_executor_backend: str = "mp" diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py b/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py index bf56df155..0401655a8 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py @@ -20,6 +20,7 @@ from vllm.config import LoadConfig from vllm.logger import init_logger from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes +from vllm_omni.diffusion.cache.cache_dit_backend import cache_summary from vllm_omni.diffusion.cache.selector import get_cache_backend from vllm_omni.diffusion.compile import regionally_compile from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig @@ -284,11 +285,19 @@ class GPUDiffusionModelRunner: req.sampling_params.generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed) # Refresh cache context if needed - if self.cache_backend is not None and self.cache_backend.is_enabled(): + if ( + not getattr(req, "skip_cache_refresh", False) + and self.cache_backend is not None + and self.cache_backend.is_enabled() + ): self.cache_backend.refresh(self.pipeline, req.sampling_params.num_inference_steps) with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config): with record_function("pipeline_forward"): output = self.pipeline.forward(req) + # NOTE: + if self.od_config.cache_backend == "cache_dit" and self.od_config.enable_cache_dit_summary: + cache_summary(self.pipeline, details=True) + return output diff --git a/vllm_omni/diffusion/worker/npu/npu_worker.py b/vllm_omni/diffusion/worker/npu/npu_worker.py index 446c29cae..c4d2c60e5 100644 --- a/vllm_omni/diffusion/worker/npu/npu_worker.py +++ b/vllm_omni/diffusion/worker/npu/npu_worker.py @@ -132,7 +132,11 @@ class NPUWorker: req.generator = torch.Generator(device=self.device).manual_seed(req.seed) # Refresh cache context if needed - if self.cache_backend is not None and self.cache_backend.is_enabled(): + if ( + not getattr(req, "skip_cache_refresh", False) + and self.cache_backend is not None + and self.cache_backend.is_enabled() + ): self.cache_backend.refresh(self.pipeline, req.num_inference_steps) with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config): output = self.pipeline.forward(req) diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index a78710fa2..179f58ed2 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -162,6 +162,7 @@ class AsyncOmni(OmniBase): "vae_use_tiling": kwargs.get("vae_use_tiling", False), "cache_backend": cache_backend, "cache_config": cache_config, + "enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False), "enable_cpu_offload": kwargs.get("enable_cpu_offload", False), "enforce_eager": kwargs.get("enforce_eager", False), }, diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index c3a37e3c8..bbd167111 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -175,6 +175,11 @@ class OmniServeCommand(CLISubcommand): default=None, help="JSON string of cache configuration (e.g., '{\"rel_l1_thresh\": 0.2}').", ) + omni_config_group.add_argument( + "--enable-cache-dit-summary", + action="store_true", + help="Enable cache-dit summary logging after diffusion forward passes.", + ) # VAE memory optimization parameters omni_config_group.add_argument( From b11d436e199867050472be1ee7e72ef706aff184 Mon Sep 17 00:00:00 2001 From: SYLAR <125541396+lishunyang12@users.noreply.github.com> Date: Wed, 28 Jan 2026 20:18:02 +0800 Subject: [PATCH 44/83] [Doc]Format profiling doc (#993) Signed-off-by: lishunyang Signed-off-by: Hongsheng Liu Co-authored-by: Hongsheng Liu Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/contributing/profiling.md | 35 +++++++++------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/docs/contributing/profiling.md b/docs/contributing/profiling.md index 4d18d0fb4..a7df8c322 100644 --- a/docs/contributing/profiling.md +++ b/docs/contributing/profiling.md @@ -2,7 +2,7 @@ > **Warning:** Profiling incurs significant overhead. Use only for development and debugging, never in production. -vLLM-Omni uses the PyTorch Profiler to analyze performance across both **Multi-Stage LLMs** and **Diffusion Models**. +vLLM-Omni uses the PyTorch Profiler to analyze performance across both **multi-stage omni-modality models** and **diffusion models**. ### 1. Set the Output Directory Before running any script, set this environment variable. The system detects this and automatically saves traces here. @@ -11,7 +11,7 @@ Before running any script, set this environment variable. The system detects thi export VLLM_TORCH_PROFILER_DIR=./profiles ``` -### 2. Start Profiling +### 2. Profiling Omni-Modality Models It is best to limit profiling to one iteration to keep trace files manageable. @@ -77,12 +77,12 @@ omni_llm.close() **Examples**: -1. **Qwen-omni 2.5**: [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py) +1. **Qwen2.5-Omni**: [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py) -2. **Qwen-omni 3.0**: [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py) +2. **Qwen3-Omni**: [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py) -**For Diffusion Models as a single stage** +### 3. Profiling diffusion models Diffusion profiling is End-to-End, capturing encoding, denoising loops, and decoding. @@ -131,27 +131,10 @@ python image_to_video.py \ 2. **Wan-AI/Wan2.2-I2V-A14B-Diffusers**: [https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video) +> **Note:** +As of now, asynchronous (online) profiling is not fully supported in vLLM-Omni. While start_profile() and stop_profile() methods exist, they are only reliable in offline inference scripts (e.g., the provided end2end.py examples). Do not use them in server-mode or streaming scenarios—traces may be incomplete or fail to flush. -**Online Inference(Async)** - -For online serving using AsyncOmni, the methods are asynchronous. This allows you to toggle profiling dynamically without restarting the server. - -```python -from vllm_omni import AsyncOmni - -# Inside an async function: -async_omni = AsyncOmni.from_engine_args(engine_args) - -await async_omni.start_profile() - -async for output in async_omni.generate(prompt, sampling_params, request_id): - # Process outputs... - pass - -await async_omni.stop_profile() -``` - -### 3. Analyzing Omni Traces +### 4. Analyzing Omni Traces Output files are saved to your configured ```VLLM_TORCH_PROFILER_DIR```. @@ -163,4 +146,4 @@ Output files are saved to your configured ```VLLM_TORCH_PROFILER_DIR```. - [Perfetto](https://ui.perfetto.dev/)(recommended) - ```chrome://tracing```(Chrome only) -**Note**: vLLM-Omni reuses the PyTorch Profiler infrastructure from vLLM. See the official vLLM profiler documentation: [vLLM Profiling Guide](https://docs.vllm.ai/en/latest/dev/profiling.html) +**Note**: vLLM-Omni reuses the PyTorch Profiler infrastructure from vLLM. See the official vLLM profiler documentation: [vLLM Profiling Guide](https://docs.vllm.ai/en/stable/contributing/profiling/) From c9392dff01982eeec9a8bd24fcc75f6a1a7fd9b2 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Wed, 28 Jan 2026 22:12:52 +0800 Subject: [PATCH 45/83] [Hardware] Support platforms and plugin system (#774) Signed-off-by: gcanlin --- .buildkite/pipeline.yml | 20 +- .buildkite/test-amd.yaml | 2 +- docs/configuration/stage_configs.md | 6 +- .../stage_configs/qwen2_5_omni.yaml | 6 +- .../image_to_image/image_edit.py | 22 +- .../image_to_image/image_to_image.md | 4 + .../image_to_video/README.md | 4 + .../image_to_video/image_to_video.py | 23 +- .../text_to_audio/text_to_audio.py | 5 +- .../offline_inference/text_to_image/README.md | 4 + .../text_to_image/gradio_demo.py | 9 +- .../text_to_image/text_to_image.py | 23 +- .../text_to_video/text_to_video.md | 4 + .../text_to_video/text_to_video.py | 23 +- .../qwen3_omni/qwen3_omni_moe_thinking.yaml | 2 +- .../diffusion/attention/test_attention_sp.py | 21 +- tests/diffusion/distributed/test_comm.py | 28 +- ...ion_worker.py => test_diffusion_worker.py} | 62 ++- .../stage_configs/npu/qwen2_5_omni_ci.yaml | 6 +- .../stage_configs/qwen2_5_omni_ci.yaml | 6 +- .../stage_configs/qwen3_omni_ci.yaml | 6 +- .../stage_configs/rocm/qwen2_5_omni_ci.yaml | 6 +- .../stage_configs/rocm/qwen3_omni_ci.yaml | 6 +- .../test_diffusion_cpu_offload.py | 6 +- .../offline_inference/test_diffusion_lora.py | 5 - .../offline_inference/test_qwen2_5_omni.py | 6 +- .../e2e/offline_inference/test_qwen3_omni.py | 4 +- .../test_sequence_parallel.py | 31 +- tests/e2e/offline_inference/test_t2i_model.py | 9 +- .../test_zimage_tensor_parallel.py | 4 +- .../stage_configs/qwen3_omni_ci.yaml | 10 +- .../stage_configs/qwen3_omni_thinker_ci.yaml | 2 +- .../stage_configs/rocm/qwen3_omni_ci.yaml | 6 +- .../test_images_generations_lora.py | 4 - tests/e2e/online_serving/test_qwen3_omni.py | 12 +- tests/e2e/stage_configs/qwen3_omni_ci.yaml | 6 +- tests/entrypoints/test_stage_utils.py | 55 +- .../diffusion/attention/backends/abstract.py | 56 +- .../attention/backends/ascend_attn.py | 92 ---- .../attention/backends/flash_attn.py | 54 +- .../diffusion/attention/backends/registry.py | 152 ++++++ .../diffusion/attention/backends/sage_attn.py | 2 +- .../diffusion/attention/backends/sdpa.py | 4 + .../diffusion/attention/backends/utils/fa.py | 4 +- vllm_omni/diffusion/attention/selector.py | 127 ++--- vllm_omni/diffusion/distributed/comm.py | 10 +- .../distributed/group_coordinator.py | 14 +- .../diffusion/distributed/parallel_state.py | 18 +- vllm_omni/diffusion/distributed/utils.py | 7 +- vllm_omni/diffusion/envs.py | 163 +----- .../diffusion/executor/multiproc_executor.py | 7 +- .../diffusion/hooks/sequence_parallel.py | 7 +- vllm_omni/diffusion/layers/adalayernorm.py | 8 + vllm_omni/diffusion/layers/custom_op.py | 14 +- vllm_omni/diffusion/layers/rope.py | 8 + .../longcat_image_transformer.py | 4 +- vllm_omni/diffusion/worker/__init__.py | 10 +- ...el_runner.py => diffusion_model_runner.py} | 2 +- ...iffusion_worker.py => diffusion_worker.py} | 30 +- vllm_omni/diffusion/worker/npu/__init__.py | 7 - vllm_omni/diffusion/worker/npu/npu_worker.py | 255 --------- vllm_omni/engine/arg_utils.py | 13 +- vllm_omni/entrypoints/omni_stage.py | 504 ++++++++++-------- vllm_omni/entrypoints/stage_utils.py | 6 +- vllm_omni/entrypoints/utils.py | 16 +- .../models/qwen2_5_omni/qwen2_5_omni.py | 4 +- .../qwen2_5_omni/qwen2_5_omni_token2wav.py | 8 +- .../models/qwen3_omni/qwen3_omni.py | 6 +- .../model_executor/stage_configs/bagel.yaml | 2 +- .../stage_configs/bagel_multiconnector.yaml | 2 +- .../stage_configs/qwen2_5_omni.yaml | 6 +- .../qwen2_5_omni_multiconnector.yaml | 6 +- .../stage_configs/qwen3_omni_moe.yaml | 6 +- .../qwen3_omni_moe_async_chunk.yaml | 6 +- .../qwen3_omni_moe_multiconnector.yaml | 6 +- .../stage_configs/qwen3_tts.yaml | 2 +- vllm_omni/platforms/__init__.py | 191 +++++++ vllm_omni/platforms/cuda/__init__.py | 6 + vllm_omni/platforms/cuda/platform.py | 101 ++++ vllm_omni/platforms/interface.py | 90 ++++ vllm_omni/platforms/npu/__init__.py | 6 + vllm_omni/platforms/npu/platform.py | 84 +++ .../npu/stage_configs}/qwen2_5_omni.yaml | 12 +- .../npu/stage_configs}/qwen3_omni_moe.yaml | 6 +- .../npu/stage_configs}/qwen3_tts.yaml | 2 +- .../npu => platforms/npu/worker}/__init__.py | 0 .../npu/worker}/npu_ar_model_runner.py | 2 +- .../npu/worker}/npu_ar_worker.py | 5 +- .../worker}/npu_generation_model_runner.py | 4 +- .../npu/worker}/npu_generation_worker.py | 5 +- .../npu/worker}/npu_model_runner.py | 0 vllm_omni/platforms/rocm/__init__.py | 6 + vllm_omni/platforms/rocm/platform.py | 97 ++++ .../rocm/stage_configs}/qwen2_5_omni.yaml | 6 +- .../rocm/stage_configs}/qwen3_omni_moe.yaml | 6 +- vllm_omni/platforms/xpu/__init__.py | 6 + vllm_omni/platforms/xpu/platform.py | 76 +++ vllm_omni/plugins/__init__.py | 81 +++ vllm_omni/utils/__init__.py | 13 - vllm_omni/utils/platform_utils.py | 58 -- vllm_omni/worker/gpu_ar_worker.py | 3 +- vllm_omni/worker/gpu_generation_worker.py | 3 +- vllm_omni/worker/mixins.py | 13 + 103 files changed, 1770 insertions(+), 1218 deletions(-) rename tests/diffusion/{test_gpu_diffusion_worker.py => test_diffusion_worker.py} (81%) delete mode 100644 vllm_omni/diffusion/attention/backends/ascend_attn.py create mode 100644 vllm_omni/diffusion/attention/backends/registry.py rename vllm_omni/diffusion/worker/{gpu_diffusion_model_runner.py => diffusion_model_runner.py} (99%) rename vllm_omni/diffusion/worker/{gpu_diffusion_worker.py => diffusion_worker.py} (94%) delete mode 100644 vllm_omni/diffusion/worker/npu/__init__.py delete mode 100644 vllm_omni/diffusion/worker/npu/npu_worker.py create mode 100644 vllm_omni/platforms/__init__.py create mode 100644 vllm_omni/platforms/cuda/__init__.py create mode 100644 vllm_omni/platforms/cuda/platform.py create mode 100644 vllm_omni/platforms/interface.py create mode 100644 vllm_omni/platforms/npu/__init__.py create mode 100644 vllm_omni/platforms/npu/platform.py rename vllm_omni/{model_executor/stage_configs/npu => platforms/npu/stage_configs}/qwen2_5_omni.yaml (89%) rename vllm_omni/{model_executor/stage_configs/npu => platforms/npu/stage_configs}/qwen3_omni_moe.yaml (93%) rename vllm_omni/{model_executor/stage_configs/npu => platforms/npu/stage_configs}/qwen3_tts.yaml (89%) rename vllm_omni/{worker/npu => platforms/npu/worker}/__init__.py (100%) rename vllm_omni/{worker/npu => platforms/npu/worker}/npu_ar_model_runner.py (99%) rename vllm_omni/{worker/npu => platforms/npu/worker}/npu_ar_worker.py (74%) rename vllm_omni/{worker/npu => platforms/npu/worker}/npu_generation_model_runner.py (99%) rename vllm_omni/{worker/npu => platforms/npu/worker}/npu_generation_worker.py (72%) rename vllm_omni/{worker/npu => platforms/npu/worker}/npu_model_runner.py (100%) create mode 100644 vllm_omni/platforms/rocm/__init__.py create mode 100644 vllm_omni/platforms/rocm/platform.py rename vllm_omni/{model_executor/stage_configs/rocm => platforms/rocm/stage_configs}/qwen2_5_omni.yaml (94%) rename vllm_omni/{model_executor/stage_configs/rocm => platforms/rocm/stage_configs}/qwen3_omni_moe.yaml (93%) create mode 100644 vllm_omni/platforms/xpu/__init__.py create mode 100644 vllm_omni/platforms/xpu/platform.py create mode 100644 vllm_omni/plugins/__init__.py delete mode 100644 vllm_omni/utils/platform_utils.py create mode 100644 vllm_omni/worker/mixins.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d1e71aba2..78a400cd8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -18,11 +18,23 @@ steps: # queue: "ascend" - label: "Simple Unit Test" - depends_on: ~ + depends_on: image-build commands: - - ".buildkite/scripts/simple_test.sh" + - pytest -v -s tests/entrypoints/ + - pytest -v -s tests/diffusion/cache/ + - pytest -v -s tests/model_executor/models/qwen2_5_omni/test_audio_length.py + - pytest -v -s tests/worker/ agents: - queue: "cpu_queue_premerge" + queue: "gpu_1_queue" + plugins: + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" - label: "Diffusion Model Test" timeout_in_minutes: 20 @@ -149,7 +161,7 @@ steps: timeout_in_minutes: 20 depends_on: image-build commands: - - pytest -s -v tests/diffusion/test_gpu_diffusion_worker.py + - pytest -s -v tests/diffusion/test_diffusion_worker.py agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 86d65f15b..08a3f7a1c 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -54,7 +54,7 @@ steps: commands: - export MIOPEN_DEBUG_CONV_DIRECT=0 - export MIOPEN_DEBUG_CONV_GEMM=0 - - pytest -s -v tests/diffusion/test_gpu_diffusion_worker.py + - pytest -s -v tests/diffusion/test_diffusion_worker.py - label: "Omni Model Test Qwen2-5-Omni" timeout_in_minutes: 15 diff --git a/docs/configuration/stage_configs.md b/docs/configuration/stage_configs.md index 18972eb79..1f1a18902 100644 --- a/docs/configuration/stage_configs.md +++ b/docs/configuration/stage_configs.md @@ -40,7 +40,7 @@ stage_args: engine_args: # Engine arguments for a certain engine model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration # The model implementation registered in model_executor/models/registry.py - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker # The specific worker used + worker_type: ar # The specific worker used scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler # The specific scehduler used gpu_memory_utilization: 0.8 # The gpu memory allocation for the stage within a single chip enforce_eager: true # Now we only support eager mode @@ -66,7 +66,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true @@ -92,7 +92,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/docs/configuration/stage_configs/qwen2_5_omni.yaml b/docs/configuration/stage_configs/qwen2_5_omni.yaml index e71f348de..e20e79c3e 100644 --- a/docs/configuration/stage_configs/qwen2_5_omni.yaml +++ b/docs/configuration/stage_configs/qwen2_5_omni.yaml @@ -8,7 +8,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true # Now we only support eager mode @@ -34,7 +34,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true @@ -60,7 +60,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index f2065e6b2..896c611b4 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -81,7 +81,7 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig, logger from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import detect_device_type, is_npu +from vllm_omni.platforms import current_omni_platform def parse_args() -> argparse.Namespace: @@ -280,6 +280,16 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Disable torch.compile and force eager execution.", ) + parser.add_argument( + "--vae_use_slicing", + action="store_true", + help="Enable VAE slicing for memory optimization.", + ) + parser.add_argument( + "--vae_use_tiling", + action="store_true", + help="Enable VAE tiling for memory optimization.", + ) parser.add_argument( "--enable-cpu-offload", action="store_true", @@ -306,12 +316,8 @@ def main(): else: input_image = input_images - device = detect_device_type() - generator = torch.Generator(device=device).manual_seed(args.seed) + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) - # Enable VAE memory optimizations on NPU - vae_use_slicing = is_npu() - vae_use_tiling = is_npu() parallel_config = DiffusionParallelConfig( ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree, @@ -344,8 +350,8 @@ def main(): # Initialize Omni with appropriate pipeline omni = Omni( model=args.model, - vae_use_slicing=vae_use_slicing, - vae_use_tiling=vae_use_tiling, + vae_use_slicing=args.vae_use_slicing, + vae_use_tiling=args.vae_use_tiling, cache_backend=args.cache_backend, cache_config=cache_config, parallel_config=parallel_config, diff --git a/examples/offline_inference/image_to_image/image_to_image.md b/examples/offline_inference/image_to_image/image_to_image.md index d624b4684..b9e25bf98 100644 --- a/examples/offline_inference/image_to_image/image_to_image.md +++ b/examples/offline_inference/image_to_image/image_to_image.md @@ -47,4 +47,8 @@ Key arguments: - `--guidance_scale`: guidance scale for guidance-distilled models (default: 1.0, disabled). Unlike classifier-free guidance (--cfg_scale), guidance-distilled models take the guidance scale directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models. - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--output`: path to save the generated PNG. +- `--vae_use_slicing`: enable VAE slicing for memory optimization. +- `--vae_use_tiling`: enable VAE tiling for memory optimization. - `--enable-cpu-offload`: enable CPU offloading for diffusion models. + +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. diff --git a/examples/offline_inference/image_to_video/README.md b/examples/offline_inference/image_to_video/README.md index afebb1963..52bb389e7 100644 --- a/examples/offline_inference/image_to_video/README.md +++ b/examples/offline_inference/image_to_video/README.md @@ -54,4 +54,8 @@ Key arguments: - `--num_inference_steps`: Number of denoising steps (default 50). - `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: Path to save the generated video. +- `--vae_use_slicing`: Enable VAE slicing for memory optimization. +- `--vae_use_tiling`: Enable VAE tiling for memory optimization. - `--enable-cpu-offload`: enable CPU offloading for diffusion models. + +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 06c13fbee..4fca7825c 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -29,7 +29,7 @@ import torch from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import detect_device_type, is_npu +from vllm_omni.platforms import current_omni_platform def parse_args() -> argparse.Namespace: @@ -59,6 +59,16 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--output", type=str, default="i2v_output.mp4", help="Path to save the video (mp4).") parser.add_argument("--fps", type=int, default=16, help="Frames per second for the output video.") + parser.add_argument( + "--vae_use_slicing", + action="store_true", + help="Enable VAE slicing for memory optimization.", + ) + parser.add_argument( + "--vae_use_tiling", + action="store_true", + help="Enable VAE tiling for memory optimization.", + ) parser.add_argument( "--enable-cpu-offload", action="store_true", @@ -80,8 +90,7 @@ def calculate_dimensions(image: PIL.Image.Image, max_area: int = 480 * 832) -> t def main(): args = parse_args() - device = detect_device_type() - generator = torch.Generator(device=device).manual_seed(args.seed) + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) # Load input image image = PIL.Image.open(args.image).convert("RGB") @@ -98,17 +107,13 @@ def main(): # Resize image to target dimensions image = image.resize((width, height), PIL.Image.Resampling.LANCZOS) - # Enable VAE memory optimizations on NPU - vae_use_slicing = is_npu() - vae_use_tiling = is_npu() - # Check if profiling is requested via environment variable profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) omni = Omni( model=args.model, - vae_use_slicing=vae_use_slicing, - vae_use_tiling=vae_use_tiling, + vae_use_slicing=args.vae_use_slicing, + vae_use_tiling=args.vae_use_tiling, boundary_ratio=args.boundary_ratio, flow_shift=args.flow_shift, enable_cpu_offload=args.enable_cpu_offload, diff --git a/examples/offline_inference/text_to_audio/text_to_audio.py b/examples/offline_inference/text_to_audio/text_to_audio.py index 1124dbdb4..0a9efcca5 100644 --- a/examples/offline_inference/text_to_audio/text_to_audio.py +++ b/examples/offline_inference/text_to_audio/text_to_audio.py @@ -22,7 +22,7 @@ import torch from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams -from vllm_omni.utils.platform_utils import detect_device_type +from vllm_omni.platforms import current_omni_platform def parse_args() -> argparse.Namespace: @@ -118,8 +118,7 @@ def save_audio(audio_data: np.ndarray, output_path: str, sample_rate: int = 4410 def main(): args = parse_args() - device = detect_device_type() - generator = torch.Generator(device=device).manual_seed(args.seed) + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) print(f"\n{'=' * 60}") print("Stable Audio Open - Text-to-Audio Generation") diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index f4de19db8..ab28b115c 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -96,8 +96,12 @@ Key arguments: - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--height/--width`: output resolution (defaults 1024x1024). - `--output`: path to save the generated PNG. +- `--vae_use_slicing`: enable VAE slicing for memory optimization. +- `--vae_use_tiling`: enable VAE tiling for memory optimization. - `--enable-cpu-offload`: enable CPU offloading for diffusion models. +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. + > ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes. ## Web UI Demo diff --git a/examples/offline_inference/text_to_image/gradio_demo.py b/examples/offline_inference/text_to_image/gradio_demo.py index 30446f50e..614e40903 100644 --- a/examples/offline_inference/text_to_image/gradio_demo.py +++ b/examples/offline_inference/text_to_image/gradio_demo.py @@ -7,7 +7,7 @@ import torch from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import detect_device_type, is_npu +from vllm_omni.platforms import current_omni_platform ASPECT_RATIOS: dict[str, tuple[int, int]] = { "1:1": (1328, 1328), @@ -62,8 +62,8 @@ def parse_args() -> argparse.Namespace: @lru_cache(maxsize=1) def get_omni(model_name: str) -> Omni: # Enable VAE memory optimizations on NPU - vae_use_slicing = is_npu() - vae_use_tiling = is_npu() + vae_use_slicing = current_omni_platform.is_npu() + vae_use_tiling = current_omni_platform.is_npu() return Omni( model=model_name, vae_use_slicing=vae_use_slicing, @@ -72,7 +72,6 @@ def get_omni(model_name: str) -> Omni: def build_demo(args: argparse.Namespace) -> gr.Blocks: - device = detect_device_type() omni = get_omni(args.model) def run_inference( @@ -99,7 +98,7 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks: raise gr.Error("Inference steps must be a positive integer.") if num_images not in {1, 2, 3, 4}: raise gr.Error("Number of images must be 1, 2, 3, or 4.") - generator = torch.Generator(device=device).manual_seed(seed) + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed) outputs = omni.generate( prompt.strip(), OmniDiffusionSamplingParams( diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index efc5010e0..3e0628a15 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -12,7 +12,7 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig, logger from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import detect_device_type, is_npu +from vllm_omni.platforms import current_omni_platform def parse_args() -> argparse.Namespace: @@ -113,17 +113,22 @@ def parse_args() -> argparse.Namespace: default=1, help="Number of GPUs used for tensor parallelism (TP) inside the DiT.", ) + parser.add_argument( + "--vae_use_slicing", + action="store_true", + help="Enable VAE slicing for memory optimization.", + ) + parser.add_argument( + "--vae_use_tiling", + action="store_true", + help="Enable VAE tiling for memory optimization.", + ) return parser.parse_args() def main(): args = parse_args() - device = detect_device_type() - generator = torch.Generator(device=device).manual_seed(args.seed) - - # Enable VAE memory optimizations on NPU - vae_use_slicing = is_npu() - vae_use_tiling = is_npu() + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) # Configure cache based on backend type cache_config = None @@ -167,8 +172,8 @@ def main(): omni = Omni( model=args.model, - vae_use_slicing=vae_use_slicing, - vae_use_tiling=vae_use_tiling, + vae_use_slicing=args.vae_use_slicing, + vae_use_tiling=args.vae_use_tiling, cache_backend=args.cache_backend, cache_config=cache_config, enable_cache_dit_summary=args.enable_cache_dit_summary, diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index 7204ec58a..cd5bd2be4 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -29,4 +29,8 @@ Key arguments: - `--boundary_ratio`: Boundary split ratio for low/high DiT. - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: path to save the generated video. +- `--vae_use_slicing`: enable VAE slicing for memory optimization. +- `--vae_use_tiling`: enable VAE tiling for memory optimization. - `--enable-cpu-offload`: enable CPU offloading for diffusion models. + +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 49522278f..464ff9cca 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -13,7 +13,7 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import detect_device_type, is_npu +from vllm_omni.platforms import current_omni_platform def parse_args() -> argparse.Namespace: @@ -54,6 +54,16 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--output", type=str, default="wan22_output.mp4", help="Path to save the video (mp4).") parser.add_argument("--fps", type=int, default=24, help="Frames per second for the output video.") + parser.add_argument( + "--vae_use_slicing", + action="store_true", + help="Enable VAE slicing for memory optimization.", + ) + parser.add_argument( + "--vae_use_tiling", + action="store_true", + help="Enable VAE tiling for memory optimization.", + ) parser.add_argument( "--enforce_eager", action="store_true", @@ -81,12 +91,7 @@ def parse_args() -> argparse.Namespace: def main(): args = parse_args() - device = detect_device_type() - generator = torch.Generator(device=device).manual_seed(args.seed) - - # Enable VAE memory optimizations on NPU - vae_use_slicing = is_npu() - vae_use_tiling = is_npu() + generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed) # Wan2.2 cache-dit tuning (from cache-dit examples and cache_alignment). cache_config = None @@ -118,8 +123,8 @@ def main(): omni = Omni( model=args.model, - vae_use_slicing=vae_use_slicing, - vae_use_tiling=vae_use_tiling, + vae_use_slicing=args.vae_use_slicing, + vae_use_tiling=args.vae_use_tiling, boundary_ratio=args.boundary_ratio, flow_shift=args.flow_shift, cache_backend=args.cache_backend, diff --git a/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml b/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml index a69bd2fe1..34b63e26e 100644 --- a/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml +++ b/examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml @@ -13,7 +13,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.9 enforce_eager: true diff --git a/tests/diffusion/attention/test_attention_sp.py b/tests/diffusion/attention/test_attention_sp.py index c0061daf9..447d1ea93 100644 --- a/tests/diffusion/attention/test_attention_sp.py +++ b/tests/diffusion/attention/test_attention_sp.py @@ -6,7 +6,6 @@ import tempfile import pytest import torch -from vllm.platforms import current_platform from vllm_omni.diffusion.attention.layer import Attention from vllm_omni.diffusion.data import ( @@ -19,15 +18,7 @@ from vllm_omni.diffusion.distributed.parallel_state import ( initialize_model_parallel, ) from vllm_omni.diffusion.forward_context import set_forward_context -from vllm_omni.utils.platform_utils import detect_device_type - -device_type = detect_device_type() -if device_type == "cuda": - torch_device = torch.cuda -elif device_type == "npu": - torch_device = torch.npu -else: - raise ValueError(f"Unsupported device type: {device_type} for this test script! Expected GPU or NPU.") +from vllm_omni.platforms import current_omni_platform def update_environment_variables(envs_dict: dict[str, str]): @@ -171,7 +162,7 @@ def test_sequence_parallel( sequence_parallel_size = ulysses_degree * ring_degree # Skip if not enough GPUs available - available_gpus = torch_device.device_count() + available_gpus = current_omni_platform.get_device_count() if available_gpus < sequence_parallel_size: pytest.skip(f"Test requires {sequence_parallel_size} GPUs but only {available_gpus} available") @@ -338,13 +329,13 @@ def ulysses_attention_on_test_model( """Run Ulysses attention test on a test model and save results for comparison.""" # Use fixed seed for reproducibility across baseline and SP runs RANDOM_SEED = 42 - current_platform.seed_everything(RANDOM_SEED) + current_omni_platform.seed_everything(RANDOM_SEED) mode_str = "Baseline (no SP)" if is_baseline else f"SP (ulysses={ulysses_degree}, ring={ring_degree})" print(f"\n[{mode_str}] Rank {local_rank}/{world_size} - Random seed set to {RANDOM_SEED}") - device = torch.device(f"{device_type}:{local_rank}") - torch_device.set_device(device) + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) torch.set_default_device(device) torch.set_default_dtype(dtype) @@ -423,7 +414,7 @@ def ulysses_attention_on_test_model( # Generate and save full input data with fixed seed # Reinitialize RNG to ensure reproducibility torch.manual_seed(42) - torch_device.manual_seed_all(42) + current_omni_platform.seed_everything(42) full_hidden_states = torch.randn( (batch_size, seq_len, hidden_size), dtype=dtype, diff --git a/tests/diffusion/distributed/test_comm.py b/tests/diffusion/distributed/test_comm.py index 0e76f27b7..3e408c10a 100644 --- a/tests/diffusion/distributed/test_comm.py +++ b/tests/diffusion/distributed/test_comm.py @@ -14,15 +14,7 @@ from vllm_omni.diffusion.distributed.parallel_state import ( init_distributed_environment, initialize_model_parallel, ) -from vllm_omni.utils.platform_utils import detect_device_type - -device_type = detect_device_type() -if device_type == "cuda": - torch_device = torch.cuda -elif device_type == "npu": - torch_device = torch.npu -else: - raise ValueError(f"Unsupported device type: {device_type} for this test script! Expected GPU or NPU.") +from vllm_omni.platforms import current_omni_platform def update_environment_variables(envs_dict: dict[str, str]): @@ -49,7 +41,7 @@ def test_4d_identity( ): """Test that two consecutive all-to-all operations return the original input.""" # Skip if not enough GPUs available - available_gpus = torch_device.device_count() + available_gpus = current_omni_platform.get_device_count() if available_gpus < world_size: pytest.skip(f"Test requires {world_size} GPUs but only {available_gpus} available") @@ -85,8 +77,8 @@ def _test_4d_identity_worker( ): """Worker function for test_4d_identity.""" # Set device - device = torch.device(f"{device_type}:{local_rank}") - torch_device.set_device(device) + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) # Set environment variables for distributed training update_environment_variables( @@ -183,7 +175,7 @@ def test_5d_identity( ): """Test that two consecutive all-to-all operations return the original input.""" # Skip if not enough GPUs available - available_gpus = torch_device.device_count() + available_gpus = current_omni_platform.get_device_count() if available_gpus < world_size: pytest.skip(f"Test requires {world_size} GPUs but only {available_gpus} available") @@ -219,8 +211,8 @@ def _test_5d_identity_worker( ): """Worker function for test_5d_identity.""" # Set device - device = torch.device(f"{device_type}:{local_rank}") - torch_device.set_device(device) + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) # Set environment variables for distributed training update_environment_variables( @@ -316,7 +308,7 @@ def test_ring_p2p( ): """Test Ring P2P communication (send_recv).""" # Skip if not enough GPUs available - available_gpus = torch_device.device_count() + available_gpus = current_omni_platform.get_device_count() if available_gpus < world_size: pytest.skip(f"Test requires {world_size} GPUs but only {available_gpus} available") @@ -339,8 +331,8 @@ def _test_ring_p2p_worker( import sys # Set device - device = torch.device(f"{device_type}:{local_rank}") - torch_device.set_device(device) + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) # Set env vars # Use a different port to avoid conflict with other tests if run in parallel diff --git a/tests/diffusion/test_gpu_diffusion_worker.py b/tests/diffusion/test_diffusion_worker.py similarity index 81% rename from tests/diffusion/test_gpu_diffusion_worker.py rename to tests/diffusion/test_diffusion_worker.py index 7a43710c8..220f210a3 100644 --- a/tests/diffusion/test_gpu_diffusion_worker.py +++ b/tests/diffusion/test_diffusion_worker.py @@ -2,9 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Unit tests for GPUDiffusionWorker class. +Unit tests for DiffusionWorker class. -This module tests the GPUDiffusionWorker implementation: +This module tests the DiffusionWorker implementation: - load_weights: Loading model weights - sleep: Putting worker into sleep mode (levels 1 and 2) - wake_up: Waking worker from sleep mode @@ -15,7 +15,7 @@ from unittest.mock import Mock, patch import pytest import torch -from vllm_omni.diffusion.worker.gpu_diffusion_worker import GPUDiffusionWorker +from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker @pytest.fixture @@ -33,18 +33,19 @@ def mock_od_config(): @pytest.fixture def mock_gpu_worker(mock_od_config): - """Create a GPUDiffusionWorker with mocked initialization.""" - with patch.object(GPUDiffusionWorker, "init_device"): - worker = GPUDiffusionWorker(local_rank=0, rank=0, od_config=mock_od_config) + """Create a DiffusionWorker with mocked initialization.""" + with patch.object(DiffusionWorker, "init_device"): + worker = DiffusionWorker(local_rank=0, rank=0, od_config=mock_od_config) # Mock the model_runner with pipeline worker.model_runner = Mock() worker.model_runner.pipeline = Mock() + worker.device = torch.device("cuda", 0) worker._sleep_saved_buffers = {} return worker -class TestGPUDiffusionWorkerLoadWeights: - """Test GPUDiffusionWorker.load_weights method.""" +class TestDiffusionWorkerLoadWeights: + """Test DiffusionWorker.load_weights method.""" def test_load_weights_calls_pipeline(self, mock_gpu_worker): """Test that load_weights delegates to model_runner.load_weights.""" @@ -75,20 +76,21 @@ class TestGPUDiffusionWorkerLoadWeights: assert result == set() -class TestGPUDiffusionWorkerSleep: - """Test GPUDiffusionWorker.sleep method.""" +class TestDiffusionWorkerSleep: + """Test DiffusionWorker.sleep method.""" - @patch("vllm_omni.diffusion.worker.gpu_diffusion_worker.torch.cuda.mem_get_info") + @patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform") @patch("vllm.device_allocator.cumem.CuMemAllocator") - def test_sleep_level_1(self, mock_allocator_class, mock_mem_info, mock_gpu_worker): + def test_sleep_level_1(self, mock_allocator_class, mock_platform, mock_gpu_worker): """Test sleep mode level 1 (offload weights only).""" # Setup memory info mocks - # Before sleep: 1GB free, 8GB total - # After sleep: 3GB free, 8GB total (freed 2GB) - mock_mem_info.side_effect = [ - (1 * 1024**3, 8 * 1024**3), # Before sleep - (3 * 1024**3, 8 * 1024**3), # After sleep + # Before sleep: 1GB free + # After sleep: 3GB free (freed 2GB) + mock_platform.get_free_memory.side_effect = [ + 1 * 1024**3, # Before sleep + 3 * 1024**3, # After sleep ] + mock_platform.get_device_total_memory.return_value = 8 * 1024**3 # Setup allocator mock mock_allocator = Mock() @@ -104,15 +106,16 @@ class TestGPUDiffusionWorkerSleep: # Verify buffers were NOT saved (level 1 doesn't save buffers) assert len(mock_gpu_worker._sleep_saved_buffers) == 0 - @patch("vllm_omni.diffusion.worker.gpu_diffusion_worker.torch.cuda.mem_get_info") + @patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform") @patch("vllm.device_allocator.cumem.CuMemAllocator") - def test_sleep_level_2(self, mock_allocator_class, mock_mem_info, mock_gpu_worker): + def test_sleep_level_2(self, mock_allocator_class, mock_platform, mock_gpu_worker): """Test sleep mode level 2 (offload all, save buffers).""" # Setup memory info mocks - mock_mem_info.side_effect = [ - (1 * 1024**3, 8 * 1024**3), # Before sleep - (5 * 1024**3, 8 * 1024**3), # After sleep (freed 4GB) + mock_platform.get_free_memory.side_effect = [ + 1 * 1024**3, # Before sleep + 5 * 1024**3, # After sleep (freed 4GB) ] + mock_platform.get_device_total_memory.return_value = 8 * 1024**3 # Setup allocator mock mock_allocator = Mock() @@ -141,15 +144,16 @@ class TestGPUDiffusionWorkerSleep: assert "buffer1" in mock_gpu_worker._sleep_saved_buffers assert "buffer2" in mock_gpu_worker._sleep_saved_buffers - @patch("vllm_omni.diffusion.worker.gpu_diffusion_worker.torch.cuda.mem_get_info") + @patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform") @patch("vllm.device_allocator.cumem.CuMemAllocator") - def test_sleep_memory_freed_validation(self, mock_allocator_class, mock_mem_info, mock_gpu_worker): + def test_sleep_memory_freed_validation(self, mock_allocator_class, mock_platform, mock_gpu_worker): """Test that sleep validates memory was actually freed.""" # Simulate memory increase (should trigger assertion error) - mock_mem_info.side_effect = [ - (3 * 1024**3, 8 * 1024**3), # Before sleep: 3GB free - (1 * 1024**3, 8 * 1024**3), # After sleep: 1GB free (negative freed!) + mock_platform.get_free_memory.side_effect = [ + 3 * 1024**3, # Before sleep: 3GB free + 1 * 1024**3, # After sleep: 1GB free (negative freed!) ] + mock_platform.get_device_total_memory.return_value = 8 * 1024**3 mock_allocator = Mock() mock_allocator_class.get_instance = Mock(return_value=mock_allocator) @@ -160,8 +164,8 @@ class TestGPUDiffusionWorkerSleep: mock_gpu_worker.sleep(level=1) -class TestGPUDiffusionWorkerWakeUp: - """Test GPUDiffusionWorker.wake_up method.""" +class TestDiffusionWorkerWakeUp: + """Test DiffusionWorker.wake_up method.""" @patch("vllm.device_allocator.cumem.CuMemAllocator") def test_wake_up_without_buffers(self, mock_allocator_class, mock_gpu_worker): diff --git a/tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml index d2aae1c0d..d6b04258a 100644 --- a/tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/npu/qwen2_5_omni_ci.yaml @@ -10,7 +10,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler max_model_len: 896 max_num_batched_tokens: 896 @@ -40,7 +40,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler max_model_len: 896 max_num_batched_tokens: 896 @@ -70,7 +70,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/tests/e2e/offline_inference/stage_configs/qwen2_5_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/qwen2_5_omni_ci.yaml index eb9109adc..e093ec51b 100644 --- a/tests/e2e/offline_inference/stage_configs/qwen2_5_omni_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/qwen2_5_omni_ci.yaml @@ -11,7 +11,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler max_model_len: 896 max_num_batched_tokens: 896 @@ -41,7 +41,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler max_model_len: 896 max_num_batched_tokens: 896 @@ -71,7 +71,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml index 0780420fd..7ac435ea5 100644 --- a/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml @@ -12,7 +12,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: false @@ -42,7 +42,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: true @@ -73,7 +73,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/tests/e2e/offline_inference/stage_configs/rocm/qwen2_5_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/rocm/qwen2_5_omni_ci.yaml index 96e9d7fa7..474df5e79 100644 --- a/tests/e2e/offline_inference/stage_configs/rocm/qwen2_5_omni_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/rocm/qwen2_5_omni_ci.yaml @@ -11,7 +11,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler max_model_len: 896 max_num_batched_tokens: 896 @@ -41,7 +41,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler max_model_len: 896 max_num_batched_tokens: 896 @@ -71,7 +71,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml index 0780420fd..7ac435ea5 100644 --- a/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml @@ -12,7 +12,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: false @@ -42,7 +42,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: true @@ -73,7 +73,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/tests/e2e/offline_inference/test_diffusion_cpu_offload.py b/tests/e2e/offline_inference/test_diffusion_cpu_offload.py index f6606dce6..35e106df8 100644 --- a/tests/e2e/offline_inference/test_diffusion_cpu_offload.py +++ b/tests/e2e/offline_inference/test_diffusion_cpu_offload.py @@ -7,7 +7,7 @@ from vllm.distributed.parallel_state import cleanup_dist_env_and_memory from tests.utils import GPUMemoryMonitor from vllm_omni.inputs.data import OmniDiffusionSamplingParams -from vllm_omni.utils.platform_utils import is_npu, is_rocm +from vllm_omni.platforms import current_omni_platform # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] @@ -20,7 +20,7 @@ models = ["riverclouds/qwen_image_random"] def inference(model_name: str, offload: bool = True): - torch.cuda.empty_cache() + current_omni_platform.empty_cache() device_index = torch.cuda.current_device() monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02) monitor.start() @@ -45,7 +45,7 @@ def inference(model_name: str, offload: bool = True): return peak -@pytest.mark.skipif(is_npu() or is_rocm(), reason="Hardware not supported") +@pytest.mark.skipif(current_omni_platform.is_npu() or current_omni_platform.is_rocm(), reason="Hardware not supported") @pytest.mark.parametrize("model_name", models) def test_cpu_offload_diffusion_model(model_name: str): try: diff --git a/tests/e2e/offline_inference/test_diffusion_lora.py b/tests/e2e/offline_inference/test_diffusion_lora.py index ef734da61..2465bc607 100644 --- a/tests/e2e/offline_inference/test_diffusion_lora.py +++ b/tests/e2e/offline_inference/test_diffusion_lora.py @@ -9,7 +9,6 @@ from safetensors.torch import save_file from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import is_npu # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] @@ -25,10 +24,6 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" # model to reduce runtime and avoid extra downloads. models = ["Tongyi-MAI/Z-Image-Turbo"] -# NPU still can't run Tongyi-MAI/Z-Image-Turbo properly. -if is_npu(): - pytest.skip("Tongyi-MAI/Z-Image-Turbo is not supported on NPU yet.", allow_module_level=True) - @pytest.mark.parametrize("model_name", models) def test_diffusion_model(model_name: str, tmp_path: Path): diff --git a/tests/e2e/offline_inference/test_qwen2_5_omni.py b/tests/e2e/offline_inference/test_qwen2_5_omni.py index 09b0306c1..3537ea127 100644 --- a/tests/e2e/offline_inference/test_qwen2_5_omni.py +++ b/tests/e2e/offline_inference/test_qwen2_5_omni.py @@ -13,7 +13,7 @@ from vllm.assets.video import VideoAsset from vllm.envs import VLLM_USE_MODELSCOPE from vllm.multimodal.image import convert_image_mode -from vllm_omni.utils import is_npu, is_rocm +from vllm_omni.platforms import current_omni_platform from .conftest import OmniRunner from .utils import create_new_process_for_each_test @@ -21,9 +21,9 @@ from .utils import create_new_process_for_each_test models = ["Qwen/Qwen2.5-Omni-3B"] # CI stage config optimized for 24GB GPU (L4/RTX3090) or NPU -if is_npu(): +if current_omni_platform.is_npu(): stage_config = str(Path(__file__).parent / "stage_configs" / "npu" / "qwen2_5_omni_ci.yaml") -elif is_rocm(): +elif current_omni_platform.is_rocm(): # ROCm stage config optimized for MI325 GPU stage_config = str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen2_5_omni_ci.yaml") else: diff --git a/tests/e2e/offline_inference/test_qwen3_omni.py b/tests/e2e/offline_inference/test_qwen3_omni.py index edd5958a0..470cdb73b 100644 --- a/tests/e2e/offline_inference/test_qwen3_omni.py +++ b/tests/e2e/offline_inference/test_qwen3_omni.py @@ -14,14 +14,14 @@ from pathlib import Path import pytest from vllm.assets.video import VideoAsset -from vllm_omni.utils import is_rocm +from vllm_omni.platforms import current_omni_platform from .conftest import OmniRunner models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] # CI stage config for 2xH100-80G GPUs or AMD GPU MI325 -if is_rocm(): +if current_omni_platform.is_rocm(): # ROCm stage config optimized for MI325 GPU stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")] else: diff --git a/tests/e2e/offline_inference/test_sequence_parallel.py b/tests/e2e/offline_inference/test_sequence_parallel.py index 367e2e59e..262c80243 100644 --- a/tests/e2e/offline_inference/test_sequence_parallel.py +++ b/tests/e2e/offline_inference/test_sequence_parallel.py @@ -30,8 +30,7 @@ if str(REPO_ROOT) not in sys.path: from vllm_omni import Omni from vllm_omni.diffusion.data import DiffusionParallelConfig -from vllm_omni.diffusion.distributed.parallel_state import device_count -from vllm_omni.diffusion.envs import get_device_name +from vllm_omni.platforms import current_omni_platform models = ["riverclouds/qwen_image_random"] @@ -63,9 +62,9 @@ def _cleanup_distributed(): gc.collect() - if torch.cuda.is_available(): - torch.cuda.empty_cache() - torch.cuda.synchronize() + if current_omni_platform.is_available(): + current_omni_platform.empty_cache() + current_omni_platform.synchronize() time.sleep(5) @@ -88,7 +87,7 @@ def _run_baseline(model_name: str, dtype: torch.dtype, attn_backend: str, height width=width, num_inference_steps=4, guidance_scale=0.0, - generator=torch.Generator(get_device_name()).manual_seed(seed), + generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed), num_outputs_per_prompt=1, ), ) @@ -125,7 +124,7 @@ def _run_sp( width=width, num_inference_steps=4, guidance_scale=0.0, - generator=torch.Generator(get_device_name()).manual_seed(seed), + generator=torch.Generator(current_omni_platform.device_type).manual_seed(seed), num_outputs_per_prompt=1, ), ) @@ -157,8 +156,8 @@ def test_baseline_only(model_name: str, dtype: torch.dtype, attn_backend: str): @pytest.mark.parametrize("attn_backend", ["sdpa"]) def test_sp_ulysses2_only(model_name: str, dtype: torch.dtype, attn_backend: str): """Test SP inference only (ulysses=2).""" - if device_count() < 2: - pytest.skip(f"Test requires 2 GPUs but only {device_count()} available") + if current_omni_platform.get_device_count() < 2: + pytest.skip(f"Test requires 2 GPUs but only {current_omni_platform.get_device_count()} available") height = 256 width = 256 @@ -177,8 +176,8 @@ def test_sp_ulysses2_only(model_name: str, dtype: torch.dtype, attn_backend: str @pytest.mark.parametrize("attn_backend", ["sdpa"]) def test_sp_ring2_only(model_name: str, dtype: torch.dtype, attn_backend: str): """Test SP inference only (ring=2).""" - if device_count() < 2: - pytest.skip(f"Test requires 2 GPUs but only {device_count()} available") + if current_omni_platform.get_device_count() < 2: + pytest.skip(f"Test requires 2 GPUs but only {current_omni_platform.get_device_count()} available") height = 256 width = 256 @@ -212,8 +211,8 @@ def test_sequence_parallel( ) sp_size = ulysses_degree * ring_degree - if device_count() < sp_size: - pytest.skip(f"Test requires {sp_size} GPUs but only {device_count()} available") + if current_omni_platform.get_device_count() < sp_size: + pytest.skip(f"Test requires {sp_size} GPUs but only {current_omni_platform.get_device_count()} available") height = 256 width = 256 @@ -243,8 +242,10 @@ def test_sequence_parallel_ulysses4(model_name: str, dtype: torch.dtype, attn_ba ulysses_degree = 4 ring_degree = 1 - if device_count() < ulysses_degree * ring_degree: - pytest.skip(f"Test requires {ulysses_degree * ring_degree} GPUs but only {device_count()} available") + if current_omni_platform.get_device_count() < ulysses_degree * ring_degree: + pytest.skip( + f"Test requires {ulysses_degree * ring_degree} GPUs but only {current_omni_platform.get_device_count()} available" + ) height = 272 width = 272 diff --git a/tests/e2e/offline_inference/test_t2i_model.py b/tests/e2e/offline_inference/test_t2i_model.py index ba6cac34c..e7351f157 100644 --- a/tests/e2e/offline_inference/test_t2i_model.py +++ b/tests/e2e/offline_inference/test_t2i_model.py @@ -7,7 +7,7 @@ import torch from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import is_npu, is_rocm +from vllm_omni.platforms import current_omni_platform # ruff: noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[2] @@ -21,12 +21,11 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" models = ["Tongyi-MAI/Z-Image-Turbo", "riverclouds/qwen_image_random"] -# NPU still can't run Tongyi-MAI/Z-Image-Turbo properly # Modelscope can't find riverclouds/qwen_image_random # TODO: When NPU support is ready, remove this branch. -if is_npu(): - models = ["Qwen/Qwen-Image"] -elif is_rocm(): +if current_omni_platform.is_npu(): + models = ["Tongyi-MAI/Z-Image-Turbo", "Qwen/Qwen-Image"] +elif current_omni_platform.is_rocm(): # TODO: When ROCm support is ready, remove this branch. # vLLM V0.11.0 has issues running riverclouds/qwen_image_random # on ROCm diff --git a/tests/e2e/offline_inference/test_zimage_tensor_parallel.py b/tests/e2e/offline_inference/test_zimage_tensor_parallel.py index ea8f0acfe..0e3b97ec3 100644 --- a/tests/e2e/offline_inference/test_zimage_tensor_parallel.py +++ b/tests/e2e/offline_inference/test_zimage_tensor_parallel.py @@ -23,7 +23,7 @@ from tests.utils import GPUMemoryMonitor from vllm_omni import Omni from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils.platform_utils import is_npu, is_rocm +from vllm_omni.platforms import current_omni_platform # os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1" os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -129,7 +129,7 @@ def _run_zimage_generate( @pytest.mark.integration def test_zimage_tensor_parallel_tp2(tmp_path: Path): - if is_npu() or is_rocm(): + if current_omni_platform.is_npu() or current_omni_platform.is_rocm(): pytest.skip("Z-Image TP e2e test is only supported on CUDA for now.") if not torch.cuda.is_available() or torch.cuda.device_count() < 2: pytest.skip("Z-Image TP=2 requires >= 2 CUDA devices.") diff --git a/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml index c5987ebba..4db11e11f 100644 --- a/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml @@ -6,13 +6,14 @@ # The following config has been verified on 2x H100-80G GPUs. stage_args: - stage_id: 0 + stage_type: llm # Use llm stage type to launch OmniLLM runtime: devices: "0,1" max_batch_size: 5 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: false @@ -20,6 +21,7 @@ stage_args: engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" enable_prefix_caching: false + max_num_batched_tokens: 32768 hf_config_name: thinker_config tensor_parallel_size: 2 load_format: dummy @@ -36,13 +38,14 @@ stage_args: repetition_penalty: 1.05 - stage_id: 1 + stage_type: llm # Use llm stage type to launch OmniLLM runtime: devices: "1" max_batch_size: 5 engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: false @@ -67,13 +70,14 @@ stage_args: stop_token_ids: [2150] - stage_id: 2 + stage_type: llm # Use llm stage type to launch OmniLLM runtime: devices: "0" max_batch_size: 1 engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/tests/e2e/online_serving/stage_configs/qwen3_omni_thinker_ci.yaml b/tests/e2e/online_serving/stage_configs/qwen3_omni_thinker_ci.yaml index 64c6b6ee3..89d03966d 100644 --- a/tests/e2e/online_serving/stage_configs/qwen3_omni_thinker_ci.yaml +++ b/tests/e2e/online_serving/stage_configs/qwen3_omni_thinker_ci.yaml @@ -7,7 +7,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: true diff --git a/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml b/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml index ef0bae0fa..0488a06c9 100644 --- a/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml +++ b/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml @@ -12,7 +12,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: false @@ -42,7 +42,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: true @@ -73,7 +73,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/tests/e2e/online_serving/test_images_generations_lora.py b/tests/e2e/online_serving/test_images_generations_lora.py index 82044cd52..11165c6d2 100644 --- a/tests/e2e/online_serving/test_images_generations_lora.py +++ b/tests/e2e/online_serving/test_images_generations_lora.py @@ -27,8 +27,6 @@ from PIL import Image from safetensors.torch import save_file from vllm.utils.network_utils import get_open_port -from vllm_omni.utils.platform_utils import is_npu - os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" MODEL = "Tongyi-MAI/Z-Image-Turbo" @@ -122,8 +120,6 @@ class OmniServer: @pytest.fixture(scope="module") def omni_server(): - if is_npu(): - pytest.skip("Tongyi-MAI/Z-Image-Turbo is not supported on NPU yet.") with OmniServer(MODEL, ["--num-gpus", "1"]) as server: yield server diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index 8564104a0..eb134ab5e 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -27,7 +27,7 @@ from tests.conftest import ( generate_synthetic_video, modify_stage_config, ) -from vllm_omni.utils import is_rocm +from vllm_omni.platforms import current_omni_platform models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] @@ -57,7 +57,7 @@ def get_chunk_config(): CHUNK_CONFIG_PATH = get_chunk_config() # CI stage config for 2xH100-80G GPUs or AMD GPU MI325 -if is_rocm(): +if current_omni_platform.is_rocm(): # ROCm stage config optimized for MI325 GPU stage_configs = [str(Path(__file__).parent / "stage_configs" / "rocm" / "qwen3_omni_ci.yaml")] else: @@ -144,7 +144,9 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.skipif(is_rocm(), reason="Test skipped on AMD environment due to known output issues") +@pytest.mark.skipif( + current_omni_platform.is_rocm(), reason="Test skipped on AMD environment due to known output issues" +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None: """ @@ -224,7 +226,9 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N assert similarity > 0.9, "The audio content is not same as the text" -@pytest.mark.skipif(is_rocm(), reason="Test skipped on AMD environment due to known output issues") +@pytest.mark.skipif( + current_omni_platform.is_rocm(), reason="Test skipped on AMD environment due to known output issues" +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None: """ diff --git a/tests/e2e/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/stage_configs/qwen3_omni_ci.yaml index 60343dfdd..d7b07ddc9 100644 --- a/tests/e2e/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/stage_configs/qwen3_omni_ci.yaml @@ -12,7 +12,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: false @@ -43,7 +43,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: false @@ -71,7 +71,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/tests/entrypoints/test_stage_utils.py b/tests/entrypoints/test_stage_utils.py index 67a2550b7..ac503639b 100644 --- a/tests/entrypoints/test_stage_utils.py +++ b/tests/entrypoints/test_stage_utils.py @@ -1,5 +1,6 @@ import os import sys +from unittest.mock import MagicMock import pytest @@ -42,6 +43,14 @@ def _make_dummy_torch(call_log): return _Torch +def _make_mock_platform(device_type: str = "cuda", env_var: str = "CUDA_VISIBLE_DEVICES"): + """Create a mock platform for testing.""" + mock_platform = MagicMock() + mock_platform.device_type = device_type + mock_platform.device_control_env_var = env_var + return mock_platform + + @pytest.mark.usefixtures("clean_gpu_memory_between_tests") def test_set_stage_devices_respects_logical_ids(monkeypatch): # Preserve an existing logical mapping and ensure devices "0,1" map through it. @@ -49,9 +58,51 @@ def test_set_stage_devices_respects_logical_ids(monkeypatch): call_log: list[int] = [] dummy_torch = _make_dummy_torch(call_log) monkeypatch.setitem(sys.modules, "torch", dummy_torch) - monkeypatch.setattr("vllm_omni.utils.detect_device_type", lambda: "cuda") - monkeypatch.setattr("vllm_omni.utils.get_device_control_env_var", lambda: "CUDA_VISIBLE_DEVICES") + + # Mock the platform at the source module where it's defined + mock_platform = _make_mock_platform(device_type="cuda", env_var="CUDA_VISIBLE_DEVICES") + monkeypatch.setattr( + "vllm_omni.platforms.current_omni_platform", + mock_platform, + ) set_stage_devices(stage_id=0, devices="0,1") assert os.environ["CUDA_VISIBLE_DEVICES"] == "6,7" + + +@pytest.mark.usefixtures("clean_gpu_memory_between_tests") +def test_set_stage_devices_npu_platform(monkeypatch): + """Test that set_stage_devices works correctly for NPU platform.""" + monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", "4,5") + call_log: list[int] = [] + + # Create NPU mock torch + class _Npu: + @staticmethod + def is_available(): + return True + + @staticmethod + def set_device(idx): + call_log.append(idx) + + @staticmethod + def device_count(): + return 2 + + class _NpuTorch: + npu = _Npu + + monkeypatch.setitem(sys.modules, "torch", _NpuTorch) + + # Mock NPU platform at the source module where it's defined + mock_platform = _make_mock_platform(device_type="npu", env_var="ASCEND_RT_VISIBLE_DEVICES") + monkeypatch.setattr( + "vllm_omni.platforms.current_omni_platform", + mock_platform, + ) + + set_stage_devices(stage_id=0, devices="0,1") + + assert os.environ["ASCEND_RT_VISIBLE_DEVICES"] == "4,5" diff --git a/vllm_omni/diffusion/attention/backends/abstract.py b/vllm_omni/diffusion/attention/backends/abstract.py index 54906efc9..d0a62bcd9 100644 --- a/vllm_omni/diffusion/attention/backends/abstract.py +++ b/vllm_omni/diffusion/attention/backends/abstract.py @@ -7,12 +7,18 @@ from typing import Generic, TypeVar import torch +from vllm_omni.platforms import current_omni_platform + class AttentionBackend(ABC): """Abstract class for diffusion attention backends.""" accept_output_buffer: bool = False + @classmethod + def supports_attention_mask(cls) -> bool: + return False + @staticmethod @abstractmethod def get_name() -> str: @@ -77,12 +83,58 @@ class AttentionImpl(ABC, Generic[T]): ) -> None: raise NotImplementedError - @abstractmethod def forward( self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - attn_metadata: T, + attn_metadata: T | None = None, + ) -> torch.Tensor: + """Dispatch to platform-specific forward implementation.""" + if current_omni_platform.is_rocm(): + return self.forward_hip(query, key, value, attn_metadata) + elif current_omni_platform.is_cuda(): + return self.forward_cuda(query, key, value, attn_metadata) + elif current_omni_platform.is_npu(): + return self.forward_npu(query, key, value, attn_metadata) + elif current_omni_platform.is_xpu(): + return self.forward_xpu(query, key, value, attn_metadata) + else: + raise NotImplementedError(f"No forward implementation for platform: {current_omni_platform}") + + def forward_cuda( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: T | None = None, ) -> torch.Tensor: raise NotImplementedError + + def forward_npu( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: T | None = None, + ) -> torch.Tensor: + raise NotImplementedError + + def forward_xpu( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: T | None = None, + ) -> torch.Tensor: + raise NotImplementedError + + def forward_hip( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: T | None = None, + ) -> torch.Tensor: + # By default, HIP ops are compatible with CUDA ops. + return self.forward_cuda(query, key, value, attn_metadata) diff --git a/vllm_omni/diffusion/attention/backends/ascend_attn.py b/vllm_omni/diffusion/attention/backends/ascend_attn.py deleted file mode 100644 index 6563385c9..000000000 --- a/vllm_omni/diffusion/attention/backends/ascend_attn.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from importlib.util import find_spec - -import torch -from vllm.logger import init_logger - -from vllm_omni.diffusion.attention.backends.abstract import ( - AttentionBackend, - AttentionImpl, - AttentionMetadata, -) - -logger = init_logger(__name__) - - -class AscendAttentionBackend(AttentionBackend): - accept_output_buffer: bool = True - - @staticmethod - def get_supported_head_sizes() -> list[int]: - return [x for x in range(1024)] # todo - - @staticmethod - def get_name() -> str: - return "ASCEND" - - @staticmethod - def get_impl_cls() -> type["AscendAttentionBackendImpl"]: - return AscendAttentionBackendImpl - - -class AscendAttentionBackendImpl(AttentionImpl): - def __init__( - self, - num_heads: int, - head_size: int, - softmax_scale: float, - causal: bool = False, - num_kv_heads: int | None = None, - prefix: str = "", - **extra_impl_args, - ) -> None: - self.causal = causal - self.softmax_scale = softmax_scale - - def forward_native( - self, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - attn_metadata: AttentionMetadata = None, - ) -> torch.Tensor: - query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value)) - attention_mask = attn_metadata.attn_mask if attn_metadata else None - - output = torch.nn.functional.scaled_dot_product_attention( - query, - key, - value, - attn_mask=attention_mask, - dropout_p=0.0, - is_causal=self.causal, - scale=self.softmax_scale, - ) - out = output.permute(0, 2, 1, 3) - return out - - def forward( - self, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, - attn_metadata: AttentionMetadata = None, - ) -> torch.Tensor: - if find_spec("mindiesd"): - from mindiesd import attention_forward - - attention_mask = attn_metadata.attn_mask if attn_metadata else None - output = attention_forward( - query, - key, - value, - attn_mask=attention_mask, - opt_mode="manual", - op_type="fused_attn_score", - layout="BNSD", - ) - return output - else: - return self.forward_native(query, key, value, attn_metadata) diff --git a/vllm_omni/diffusion/attention/backends/flash_attn.py b/vllm_omni/diffusion/attention/backends/flash_attn.py index e9102b1d3..a9f2d925c 100644 --- a/vllm_omni/diffusion/attention/backends/flash_attn.py +++ b/vllm_omni/diffusion/attention/backends/flash_attn.py @@ -9,13 +9,6 @@ from vllm_omni.diffusion.attention.backends.abstract import ( AttentionImpl, AttentionMetadata, ) -from vllm_omni.diffusion.attention.backends.utils.fa import ( - _pad_input, - _unpad_input, - _upad_input, - flash_attn_func, - flash_attn_varlen_func, -) logger = init_logger(__name__) @@ -23,6 +16,10 @@ logger = init_logger(__name__) class FlashAttentionBackend(AttentionBackend): accept_output_buffer: bool = True + @classmethod + def supports_attention_mask(cls) -> bool: + return True + @staticmethod def get_supported_head_sizes() -> list[int]: return [64, 96, 128, 192, 256] @@ -51,25 +48,22 @@ class FlashAttentionImpl(AttentionImpl): self.causal = causal self.softmax_scale = softmax_scale - def forward( + def forward_cuda( self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, attn_metadata: AttentionMetadata = None, ) -> torch.Tensor: - """ - Flash attention implementation. + """CUDA/ROCm flash attention implementation.""" + from vllm_omni.diffusion.attention.backends.utils.fa import ( + _pad_input, + _unpad_input, + _upad_input, + flash_attn_func, + flash_attn_varlen_func, + ) - Args: - query: (batch_size, seq_len, num_heads, head_dim) - key: (batch_size, seq_len, num_heads, head_dim) - value: (batch_size, seq_len, num_heads, head_dim) - attn_metadata: AttentionMetadata. Attention mask is supported as attn_metadata.attn_mask - - Returns: - (batch_size, seq_len, num_heads, head_dim) - """ query_length = query.size(1) attention_mask = attn_metadata.attn_mask if attn_metadata is not None else None # Contains at least one padding token in the sequence @@ -106,3 +100,25 @@ class FlashAttentionImpl(AttentionImpl): softmax_scale=self.softmax_scale, ) return out + + def forward_npu( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: AttentionMetadata = None, + ) -> torch.Tensor: + """NPU attention implementation using mindiesd.""" + from mindiesd import attention_forward + + attention_mask = attn_metadata.attn_mask if attn_metadata else None + output = attention_forward( + query, + key, + value, + attn_mask=attention_mask, + opt_mode="manual", + op_type="fused_attn_score", + layout="BNSD", + ) + return output diff --git a/vllm_omni/diffusion/attention/backends/registry.py b/vllm_omni/diffusion/attention/backends/registry.py new file mode 100644 index 000000000..a77223d16 --- /dev/null +++ b/vllm_omni/diffusion/attention/backends/registry.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Diffusion attention backend registry. + +This module provides an enum-based registry for diffusion attention backends, +similar to vLLM's AttentionBackendEnum. Each backend registers its class path, +and platforms can override or extend backends using register_backend(). +""" + +from collections.abc import Callable +from enum import Enum, EnumMeta +from typing import TYPE_CHECKING + +from vllm.logger import init_logger +from vllm.utils.import_utils import resolve_obj_by_qualname + +if TYPE_CHECKING: + from vllm_omni.diffusion.attention.backends.abstract import AttentionBackend + +logger = init_logger(__name__) + + +class _DiffusionBackendEnumMeta(EnumMeta): + """Metaclass for DiffusionAttentionBackendEnum to provide better error messages.""" + + def __getitem__(cls, name: str) -> "DiffusionAttentionBackendEnum": + """Get backend by name with helpful error messages.""" + try: + return super().__getitem__(name) # type: ignore[return-value] + except KeyError: + members = list(cls.__members__.keys()) + valid_backends = ", ".join(members) + raise ValueError( + f"Unknown diffusion attention backend: '{name}'. Valid options are: {valid_backends}" + ) from None + + +class DiffusionAttentionBackendEnum(Enum, metaclass=_DiffusionBackendEnumMeta): + """Enumeration of all supported diffusion attention backends. + + The enum value is the default class path, but this can be overridden + at runtime using register_backend(). + + To get the actual backend class (respecting overrides), use: + backend.get_class() + + Example: + # Get backend class + backend = DiffusionAttentionBackendEnum.FLASH_ATTN + backend_cls = backend.get_class() + + # Register custom backend + @register_diffusion_backend(DiffusionAttentionBackendEnum.CUSTOM) + class MyCustomBackend: + ... + """ + + # Common backends (available on most platforms) + FLASH_ATTN = "vllm_omni.diffusion.attention.backends.flash_attn.FlashAttentionBackend" + TORCH_SDPA = "vllm_omni.diffusion.attention.backends.sdpa.SDPABackend" + SAGE_ATTN = "vllm_omni.diffusion.attention.backends.sage_attn.SageAttentionBackend" + + def get_path(self, include_classname: bool = True) -> str: + """Get the class path for this backend (respects overrides). + + Returns: + The fully qualified class path string + + Raises: + ValueError: If backend has empty path and is not registered + """ + path = _DIFFUSION_ATTN_OVERRIDES.get(self, self.value) + if not path: + raise ValueError( + f"Backend {self.name} must be registered before use. " + f"Use register_diffusion_backend(DiffusionAttentionBackendEnum.{self.name}, " + f"'your.module.YourClass')" + ) + if not include_classname: + path = path.rsplit(".", 1)[0] + return path + + def get_class(self) -> "type[AttentionBackend]": + """Get the backend class (respects overrides). + + Returns: + The backend class + + Raises: + ImportError: If the backend class cannot be imported + ValueError: If backend has empty path and is not registered + """ + return resolve_obj_by_qualname(self.get_path()) + + def is_overridden(self) -> bool: + """Check if this backend has been overridden. + + Returns: + True if the backend has a registered override + """ + return self in _DIFFUSION_ATTN_OVERRIDES + + def clear_override(self) -> None: + """Clear any override for this backend, reverting to the default.""" + _DIFFUSION_ATTN_OVERRIDES.pop(self, None) + + +# Override registry +_DIFFUSION_ATTN_OVERRIDES: dict[DiffusionAttentionBackendEnum, str] = {} + + +def register_diffusion_backend( + backend: DiffusionAttentionBackendEnum, + class_path: str | None = None, +) -> Callable[[type], type]: + """Register or override a diffusion backend implementation. + + Args: + backend: The DiffusionAttentionBackendEnum member to register + class_path: Optional class path. If not provided and used as + decorator, will be auto-generated from the class. + + Returns: + Decorator function if class_path is None, otherwise a no-op + + Examples: + # Override an existing backend + @register_diffusion_backend(DiffusionAttentionBackendEnum.FLASH_ATTN) + class MyCustomFlashAttn: + ... + + # Override an existing backend (e.g., ASCEND_ATTN) + @register_diffusion_backend(DiffusionAttentionBackendEnum.ASCEND_ATTN) + class CustomAscendAttentionBackend: + ... + + # Direct registration + register_diffusion_backend( + DiffusionAttentionBackendEnum.CUSTOM, + "my.module.MyCustomBackend" + ) + """ + + def decorator(cls: type) -> type: + _DIFFUSION_ATTN_OVERRIDES[backend] = f"{cls.__module__}.{cls.__qualname__}" + return cls + + if class_path is not None: + _DIFFUSION_ATTN_OVERRIDES[backend] = class_path + return lambda x: x + + return decorator diff --git a/vllm_omni/diffusion/attention/backends/sage_attn.py b/vllm_omni/diffusion/attention/backends/sage_attn.py index bfc06a839..af34a0633 100644 --- a/vllm_omni/diffusion/attention/backends/sage_attn.py +++ b/vllm_omni/diffusion/attention/backends/sage_attn.py @@ -54,7 +54,7 @@ class SageAttentionImpl(AttentionImpl): self.causal = causal self.softmax_scale = softmax_scale - def forward( + def forward_cuda( self, query: torch.Tensor, key: torch.Tensor, diff --git a/vllm_omni/diffusion/attention/backends/sdpa.py b/vllm_omni/diffusion/attention/backends/sdpa.py index 89b3e84a7..16bc5fcf8 100644 --- a/vllm_omni/diffusion/attention/backends/sdpa.py +++ b/vllm_omni/diffusion/attention/backends/sdpa.py @@ -16,6 +16,10 @@ logger = init_logger(__name__) class SDPABackend(AttentionBackend): accept_output_buffer: bool = True + @classmethod + def supports_attention_mask(cls) -> bool: + return True + @staticmethod def get_supported_head_sizes() -> list[int]: return [x for x in range(1024)] # todo diff --git a/vllm_omni/diffusion/attention/backends/utils/fa.py b/vllm_omni/diffusion/attention/backends/utils/fa.py index 6a02435a7..ea12bc0f1 100644 --- a/vllm_omni/diffusion/attention/backends/utils/fa.py +++ b/vllm_omni/diffusion/attention/backends/utils/fa.py @@ -15,9 +15,9 @@ import torch import torch.nn.functional as F -from vllm_omni.utils.platform_utils import is_rocm +from vllm_omni.platforms import current_omni_platform -if is_rocm(): +if current_omni_platform.is_rocm(): from vllm._aiter_ops import is_aiter_found_and_supported # Choose to enable this by default on ROCm diff --git a/vllm_omni/diffusion/attention/selector.py b/vllm_omni/diffusion/attention/selector.py index ed5946098..920396e66 100644 --- a/vllm_omni/diffusion/attention/selector.py +++ b/vllm_omni/diffusion/attention/selector.py @@ -1,52 +1,54 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Diffusion attention backend selector. + +This module provides the interface for selecting diffusion attention backends. +The actual backend selection logic is delegated to the platform layer +(vllm_omni.platforms), similar to how vLLM handles attention backend selection. + +Usage: + from vllm_omni.diffusion.attention.selector import get_attn_backend + + # Get the appropriate backend for current platform + backend_cls = get_attn_backend(head_size=64) + + # Or override via environment variable + # export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN +""" import importlib import os from functools import cache -import torch from vllm.logger import init_logger from vllm_omni.diffusion.attention.backends.abstract import ( AttentionBackend, ) -from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend -from vllm_omni.utils.platform_utils import detect_device_type, is_rocm logger = init_logger(__name__) -# environment variable value -> backend module and class -_BACKEND_CONFIG = { - "FLASH_ATTN": { - "module": "vllm_omni.diffusion.attention.backends.flash_attn", - "class": "FlashAttentionBackend", - }, - "TORCH_SDPA": { - "module": "vllm_omni.diffusion.attention.backends.sdpa", - "class": "SDPABackend", - }, - "SAGE_ATTN": { - "module": "vllm_omni.diffusion.attention.backends.sage_attn", - "class": "SageAttentionBackend", - }, - "ASCEND": {"module": "vllm_omni.diffusion.attention.backends.ascend_attn", "class": "AscendAttentionBackend"}, -} -_BACKENDS_SUPPORT_ATTENTION_MASK = ["SDPA", "ASCEND", "FLASH_ATTN"] +def _load_backend_cls(cls_path: str) -> type[AttentionBackend]: + """Load a backend class from its fully qualified path. + Args: + cls_path: Fully qualified class path (e.g., + "vllm_omni.diffusion.attention.backends.sdpa.SDPABackend") -def load_backend(backend_name: str) -> type[AttentionBackend]: - config = _BACKEND_CONFIG[backend_name] - + Returns: + The loaded backend class + """ + module_path, class_name = cls_path.rsplit(".", 1) try: - module = importlib.import_module(config["module"]) - backend_class = getattr(module, config["class"]) + module = importlib.import_module(module_path) + backend_class = getattr(module, class_name) return backend_class except ImportError as e: - raise ImportError(f"Failed to import module {config['module']}: {e}") + raise ImportError(f"Failed to import module {module_path}: {e}") except AttributeError as e: - raise AttributeError(f"Class {config['class']} not found in module: {e}") + raise AttributeError(f"Class {class_name} not found in module: {e}") @cache @@ -54,65 +56,30 @@ def get_attn_backend(head_size: int) -> type[AttentionBackend]: """ Get attention backend for diffusion models. - The backend is selected based on the following priority: - 1. DIFFUSION_ATTENTION_BACKEND environment variable (if set, e.g. export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN) - 2. Default backend (SDPA) + The backend selection is delegated to the current platform + (vllm_omni.platforms.current_omni_platform), which selects the + appropriate backend based on: + 1. User override via DIFFUSION_ATTENTION_BACKEND environment variable + 2. Platform-specific defaults and capabilities + + This is similar to how vLLM's get_attn_backend_cls works, where the + platform layer decides which backend to use based on hardware capabilities. Args: - head_size: Head size (currently not used for selection, but kept for API compatibility) + head_size: Head size for attention computation (may affect backend selection) Returns: The selected attention backend class """ - # Check environment variable + from vllm_omni.platforms import current_omni_platform - backend_name = os.environ.get("DIFFUSION_ATTENTION_BACKEND", None) + # Check environment variable for user override + selected_backend = os.environ.get("DIFFUSION_ATTENTION_BACKEND") - if detect_device_type() == "cuda" and not is_rocm(): - compute_capability = torch.cuda.get_device_capability() - major, minor = compute_capability - if 80 <= major * 10 + minor < 100: - if backend_name is None: - backend_name = "FLASH_ATTN" - else: - if backend_name == "FLASH_ATTN": - logger.warning( - """Flash Attention requires GPU with compute capability >= 8.0 or < 10.0. " - "Falling back to TORCH_SDPA backend.""" - ) - backend_name = "TORCH_SDPA" - elif detect_device_type() == "cuda" and is_rocm(): - from vllm._aiter_ops import is_aiter_found_and_supported + # Delegate to platform for backend selection + backend_cls_path = current_omni_platform.get_diffusion_attn_backend_cls( + selected_backend=selected_backend, + head_size=head_size, + ) - compute_capability = torch.cuda.get_device_capability() - major, minor = compute_capability - - # Choose to enable this by default on ROCm - # Whenever possible as it is the fastest backend - # is_aiter_found_and_supported() checks if aiter library is found - # and is aiter supported on the current platform - # aiter currently only is supported on gfx942 and gfx950 - # https://github.com/vllm-project/vllm/blob/main/vllm/_aiter_ops.py - if is_aiter_found_and_supported() and 90 < major * 10 + minor < 100: - if backend_name is None: - backend_name = "FLASH_ATTN" - else: - if backend_name == "FLASH_ATTN": - logger.warning( - "Flash Attention requires `aiter` library which is only supported " - "on gfx942 and gfx950. Falling back to TORCH_SDPA backend." - ) - backend_name = "TORCH_SDPA" - - if backend_name is not None: - backend_name_upper = backend_name.upper() - if backend_name_upper not in _BACKEND_CONFIG: - valid_backends = list(_BACKEND_CONFIG.keys()) - raise ValueError( - f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}" - ) - logger.info(f"Using attention backend '{backend_name_upper}' for diffusion") - return load_backend(backend_name_upper) - - # Default to SDPA - return SDPABackend + return _load_backend_cls(backend_cls_path) diff --git a/vllm_omni/diffusion/distributed/comm.py b/vllm_omni/diffusion/distributed/comm.py index ada03f4a1..f73b8442f 100644 --- a/vllm_omni/diffusion/distributed/comm.py +++ b/vllm_omni/diffusion/distributed/comm.py @@ -8,6 +8,8 @@ import torch import torch.distributed as dist from torch import Tensor +from vllm_omni.platforms import current_omni_platform + __all__ = ["all_to_all_4D", "all_to_all_5D", "SeqAllToAll4D", "SeqAllToAll5D", "RingComm"] @@ -48,7 +50,7 @@ def all_to_all_4D( if seq_world_size > 1: dist.all_to_all_single(output, input_t, group=group) if use_sync: - torch.cuda.synchronize() + current_omni_platform.synchronize() else: output = input_t # if scattering the seq-dim, transpose the heads back to the original dimension @@ -83,7 +85,7 @@ def all_to_all_4D( if seq_world_size > 1: dist.all_to_all_single(output, input_t, group=group) if use_sync: - torch.cuda.synchronize() + current_omni_platform.synchronize() else: output = input_t @@ -155,7 +157,7 @@ def all_to_all_5D( if seq_world_size > 1: dist.all_to_all_single(output, input_t, group=group) if use_sync: - torch.cuda.synchronize() + current_omni_platform.synchronize() else: output = input_t @@ -190,7 +192,7 @@ def all_to_all_5D( if seq_world_size > 1: dist.all_to_all_single(output, input_t, group=group) if use_sync: - torch.cuda.synchronize() + current_omni_platform.synchronize() else: output = input_t diff --git a/vllm_omni/diffusion/distributed/group_coordinator.py b/vllm_omni/diffusion/distributed/group_coordinator.py index 8e33d6fb6..b722f61c0 100644 --- a/vllm_omni/diffusion/distributed/group_coordinator.py +++ b/vllm_omni/diffusion/distributed/group_coordinator.py @@ -9,18 +9,14 @@ from typing import Any import torch import torch.distributed -from torch.cuda import synchronize from torch.distributed import Backend, ProcessGroup from vllm.logger import init_logger from vllm_omni.diffusion import envs +from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) -if envs._is_npu(): - logger.info("torch.npu synchronize") - from torch.npu import synchronize - TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"]) @@ -124,7 +120,7 @@ class GroupCoordinator: assert self.cpu_group is not None assert self.device_group is not None - self.device = envs.get_device(local_rank) + self.device = current_omni_platform.get_torch_device(local_rank) @property def first_rank(self): @@ -618,7 +614,7 @@ class PipelineGroupCoordinator(GroupCoordinator): assert self.cpu_group is not None assert self.device_group is not None - self.device = envs.get_device(local_rank) + self.device = current_omni_platform.get_torch_device(local_rank) self.recv_buffer_set: bool = False self.recv_tasks_queue: list[tuple[str, int]] = [] @@ -768,7 +764,7 @@ class PipelineGroupCoordinator(GroupCoordinator): # To protect against race condition when using batch_isend_irecv(). # should take this out once the bug with batch_isend_irecv is resolved. - synchronize() + current_omni_platform.synchronize() ops = [] recv_prev_shape_tensor = None @@ -799,7 +795,7 @@ class PipelineGroupCoordinator(GroupCoordinator): for req in reqs: req.wait() - synchronize() + current_omni_platform.synchronize() recv_prev_shape = [0, 0, 0] if recv_prev_shape_tensor is not None: diff --git a/vllm_omni/diffusion/distributed/parallel_state.py b/vllm_omni/diffusion/distributed/parallel_state.py index b24951590..2f7c2f852 100644 --- a/vllm_omni/diffusion/distributed/parallel_state.py +++ b/vllm_omni/diffusion/distributed/parallel_state.py @@ -35,6 +35,7 @@ from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm_omni.diffusion import envs +from vllm_omni.platforms import current_omni_platform from .group_coordinator import ( GroupCoordinator, @@ -42,14 +43,6 @@ from .group_coordinator import ( SequenceParallelGroupCoordinator, ) -if envs._is_npu(): - from torch.npu import device_count, set_device -elif envs._is_musa(): - from torch_musa.core.device import device_count, set_device -else: - from torch.cuda import device_count, set_device - - env_info = envs.PACKAGES_CHECKER.get_packages_info() HAS_FLASH_ATTN = env_info["has_flash_attn"] @@ -396,7 +389,7 @@ def init_distributed_environment( backend: str | None = None, ): if backend is None: - backend = envs.get_torch_distributed_backend() + backend = current_omni_platform.dist_backend logger.debug( "world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s", world_size, @@ -416,7 +409,8 @@ def init_distributed_environment( world_size=world_size, rank=rank, ) - set_device(torch.distributed.get_rank() % device_count()) + device_id = torch.distributed.get_rank() % current_omni_platform.get_device_count() + current_omni_platform.set_device(current_omni_platform.get_torch_device(device_id)) # set the local rank # local_rank is not available in torch ProcessGroup, # see https://github.com/pytorch/pytorch/issues/122816 @@ -572,7 +566,7 @@ def initialize_model_parallel( backend: str | None = None, ) -> None: if backend is None: - backend = envs.get_torch_distributed_backend() + backend = current_omni_platform.dist_backend """ Initialize model parallel groups. @@ -632,7 +626,7 @@ def initialize_model_parallel( # FIXME: Since the async p2p communication operation of NPU is not same as cuda in torch, # the pipefusion is not ready for npu yet - if envs._is_npu(): + if current_omni_platform.is_npu(): assert pipeline_parallel_size == 1, "Current pipefusion is not ready for NPU" dit_parallel_size = ( diff --git a/vllm_omni/diffusion/distributed/utils.py b/vllm_omni/diffusion/distributed/utils.py index 2e2f0fa53..92d00ef4b 100644 --- a/vllm_omni/diffusion/distributed/utils.py +++ b/vllm_omni/diffusion/distributed/utils.py @@ -5,11 +5,10 @@ import os import torch -from vllm_omni.utils.platform_utils import detect_device_type +from vllm_omni.platforms import current_omni_platform def get_local_device() -> torch.device: """Return the torch device for the current rank based on detected device type.""" - device_type = detect_device_type() - local_rank = os.environ.get("LOCAL_RANK", 0) - return torch.device(f"{device_type}:{local_rank}") + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + return current_omni_platform.get_torch_device(local_rank) diff --git a/vllm_omni/diffusion/envs.py b/vllm_omni/diffusion/envs.py index f717f2ca0..be2b58ae8 100644 --- a/vllm_omni/diffusion/envs.py +++ b/vllm_omni/diffusion/envs.py @@ -5,20 +5,15 @@ import os from collections.abc import Callable from typing import TYPE_CHECKING, Any -import torch -from packaging import version from vllm.logger import init_logger -logger = init_logger(__name__) +from vllm_omni.platforms import current_omni_platform if TYPE_CHECKING: MASTER_ADDR: str = "" MASTER_PORT: int | None = None CUDA_HOME: str | None = None LOCAL_RANK: int = 0 - CUDA_VISIBLE_DEVICES: str | None = None - CUDA_VERSION: version.Version - TORCH_VERSION: version.Version environment_variables: dict[str, Callable[[], Any]] = { # ================== Runtime Env Vars ================== @@ -32,127 +27,14 @@ environment_variables: dict[str, Callable[[], Any]] = { # local rank of the process in the distributed setting, used to determine # the GPU device id "LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")), - # used to control the visible devices in the distributed setting - "CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None), } - -def _is_hip(): - has_rocm = torch.version.hip is not None - return has_rocm - - -def _is_cuda(): - has_cuda = torch.version.cuda is not None - return has_cuda - - -def _is_musa(): - try: - if hasattr(torch, "musa") and torch.musa.is_available(): - return True - except ModuleNotFoundError: - return False - - -def _is_mps(): - return torch.backends.mps.is_available() - - -def _is_npu(): - try: - if hasattr(torch, "npu") and torch.npu.is_available(): - return True - except ModuleNotFoundError: - return False - - -def get_device(local_rank: int) -> torch.device: - if _is_cuda() or _is_hip(): - return torch.device("cuda", local_rank) - elif _is_musa(): - return torch.device("musa", local_rank) - elif _is_mps(): - return torch.device("mps") - elif _is_npu(): - return torch.device("npu", local_rank) - else: - return torch.device("cpu") - - -def get_device_name() -> str: - if _is_cuda() or _is_hip(): - return "cuda" - elif _is_musa(): - return "musa" - elif _is_mps(): - return "mps" - elif _is_npu(): - return "npu" - else: - return "cpu" - - -def get_device_version(): - if _is_hip(): - hip_version = torch.version.hip - hip_version = hip_version.split("-")[0] - return hip_version - elif _is_cuda(): - return torch.version.cuda - elif _is_musa(): - return torch.version.musa - elif _is_mps(): - return None - elif _is_npu(): - return None - else: - raise NotImplementedError("No Accelerators(AMD/NV/MTT GPU, AMD MI instinct accelerators) available") - - -def get_torch_distributed_backend() -> str: - if _is_cuda() or _is_hip(): - return "nccl" - elif _is_musa(): - return "mccl" - elif _is_mps(): - return "gloo" - elif _is_npu(): - return "hccl" - else: - raise NotImplementedError("No Accelerators(AMD/NV/MTT GPU, AMD MI instinct accelerators) available") - - -variables: dict[str, Callable[[], Any]] = { - # ================== Other Vars ================== - # used in version checking - "CUDA_VERSION": lambda: version.parse(get_device_version() or "0.0"), - "TORCH_VERSION": lambda: version.parse(version.parse(torch.__version__).base_version), -} - - -def _setup_musa(environment_variables, variables): - musa = getattr(torch, "musa", None) - if musa is None: - return - try: - if musa.is_available(): - environment_variables["MUSA_HOME"] = lambda: os.environ.get("MUSA_HOME", None) - environment_variables["MUSA_VISIBLE_DEVICES"] = lambda: os.environ.get("MUSA_VISIBLE_DEVICES", None) - musa_ver = getattr(getattr(torch, "version", None), "musa", None) - if musa_ver: - variables["MUSA_VERSION"] = lambda: version.parse(musa_ver) - except Exception: - pass - - -try: - _setup_musa(environment_variables, variables) -except (AttributeError, ModuleNotFoundError): - pass +logger = init_logger(__name__) class PackagesEnvChecker: + """Singleton class for checking package availability.""" + _instance = None def __new__(cls): @@ -163,38 +45,39 @@ class PackagesEnvChecker: def initialize(self): packages_info = {} - packages_info["has_flash_attn"] = self.check_flash_attn(packages_info) + packages_info["has_flash_attn"] = self._check_flash_attn(packages_info) self.packages_info = packages_info - def check_flash_attn(self, packages_info): - if not torch.cuda.is_available(): + def _check_flash_attn(self, packages_info) -> bool: + """Check if flash attention is available and compatible.""" + platform = current_omni_platform + + # Flash attention requires CUDA-like platforms (CUDA or ROCm) + if not platform.is_cuda_alike(): return False - # Check if torch_npu is available - if _is_npu(): - logger.info("`flash_attn` is not ready on torch_npu for now") + # Check if devices are available + if platform.get_device_count() == 0: return False - if _is_musa(): - logger.info("Flash Attention library is not supported on MUSA for the moment.") - return False try: - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - gpu_name = torch.cuda.get_device_name(device) + gpu_name = platform.get_device_name() + # Turing/Tesla/T4 GPUs don't support flash attention well if "Turing" in gpu_name or "Tesla" in gpu_name or "T4" in gpu_name: return False - else: - from flash_attn import __version__ - if __version__ < "2.6.0": - raise ImportError("install flash_attn >= 2.6.0") - return True + from flash_attn import __version__ + + if __version__ < "2.6.0": + raise ImportError("install flash_attn >= 2.6.0") + return True except ImportError: if not packages_info.get("has_aiter", False): logger.warning('Flash Attention library "flash_attn" not found, using pytorch attention implementation') return False - def get_packages_info(self): + def get_packages_info(self) -> dict: + """Get the packages info dictionary.""" return self.packages_info @@ -205,8 +88,6 @@ def __getattr__(name): # lazy evaluation of environment variables if name in environment_variables: return environment_variables[name]() - if name in variables: - return variables[name]() raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm_omni/diffusion/executor/multiproc_executor.py b/vllm_omni/diffusion/executor/multiproc_executor.py index 21a94acd3..a421a01ed 100644 --- a/vllm_omni/diffusion/executor/multiproc_executor.py +++ b/vllm_omni/diffusion/executor/multiproc_executor.py @@ -10,7 +10,7 @@ from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, DiffusionOutput from vllm_omni.diffusion.executor.abstract import DiffusionExecutor from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.scheduler import Scheduler -from vllm_omni.utils.platform_utils import get_diffusion_worker_class +from vllm_omni.diffusion.worker import WorkerProc logger = init_logger(__name__) @@ -77,9 +77,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): mp.set_start_method("spawn", force=True) processes = [] - # Get the appropriate worker class for current device - worker_proc = get_diffusion_worker_class() - # Launch all worker processes scheduler_pipe_readers = [] scheduler_pipe_writers = [] @@ -88,7 +85,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor): reader, writer = mp.Pipe(duplex=False) scheduler_pipe_writers.append(writer) process = mp.Process( - target=worker_proc.worker_main, + target=WorkerProc.worker_main, args=( i, # rank od_config, diff --git a/vllm_omni/diffusion/hooks/sequence_parallel.py b/vllm_omni/diffusion/hooks/sequence_parallel.py index 21f4a6fe1..8f2c2b755 100644 --- a/vllm_omni/diffusion/hooks/sequence_parallel.py +++ b/vllm_omni/diffusion/hooks/sequence_parallel.py @@ -335,10 +335,7 @@ class SequenceParallelSplitHook(ModelHook): 2. Creates an attention mask indicating valid vs padding positions 3. Stores the mask and padding info in ForwardContext """ - from vllm_omni.diffusion.attention.selector import ( - _BACKENDS_SUPPORT_ATTENTION_MASK, - get_attn_backend, - ) + from vllm_omni.diffusion.attention.selector import get_attn_backend from vllm_omni.diffusion.distributed.parallel_state import ( get_ring_parallel_world_size, get_sequence_parallel_rank, @@ -359,7 +356,7 @@ class SequenceParallelSplitHook(ModelHook): # Check backend compatibility attn_backend = get_attn_backend(-1) - if attn_backend.get_name() not in _BACKENDS_SUPPORT_ATTENTION_MASK: + if not attn_backend.supports_attention_mask: raise ValueError( f"Sequence length ({seq_len}) is not divisible by SP world size ({world_size}). " f"Cannot use {attn_backend.get_name()} which does not support attention_mask. " diff --git a/vllm_omni/diffusion/layers/adalayernorm.py b/vllm_omni/diffusion/layers/adalayernorm.py index 835ee07c0..9a644e1d9 100644 --- a/vllm_omni/diffusion/layers/adalayernorm.py +++ b/vllm_omni/diffusion/layers/adalayernorm.py @@ -91,6 +91,14 @@ class AdaLayerNorm(CustomOp): return output, gate_result + def forward_xpu( + self, + x: torch.Tensor, + mod_params: torch.Tensor, + index: torch.Tensor = None, + ) -> torch.Tensor: + return self.forward_native(x, mod_params, index) + def forward_native( self, x: torch.Tensor, diff --git a/vllm_omni/diffusion/layers/custom_op.py b/vllm_omni/diffusion/layers/custom_op.py index 2f50073cc..321bcbf8a 100644 --- a/vllm_omni/diffusion/layers/custom_op.py +++ b/vllm_omni/diffusion/layers/custom_op.py @@ -3,7 +3,7 @@ from typing import Any import torch.nn as nn -from vllm_omni.utils.platform_utils import detect_device_type, is_npu, is_rocm +from vllm_omni.platforms import current_omni_platform class CustomOp(nn.Module): @@ -14,16 +14,17 @@ class CustomOp(nn.Module): def __init__(self) -> None: super().__init__() - self.is_cuda = detect_device_type() == "cuda" self._forward_method = self.dispatch_forward() def dispatch_forward(self) -> Callable: - if is_rocm(): + if current_omni_platform.is_rocm(): return self.forward_hip - elif self.is_cuda: + elif current_omni_platform.is_cuda(): return self.forward_cuda - elif is_npu(): + elif current_omni_platform.is_npu(): return self.forward_npu + elif current_omni_platform.is_xpu(): + return self.forward_xpu else: return self.forward_native @@ -44,6 +45,9 @@ class CustomOp(nn.Module): def forward_npu(self, *args, **kwargs): raise NotImplementedError + def forward_xpu(self, *args, **kwargs): + raise NotImplementedError + def forward_hip(self, *args, **kwargs): # By default, we assume that HIP ops are compatible with CUDA ops. return self.forward_cuda(*args, **kwargs) diff --git a/vllm_omni/diffusion/layers/rope.py b/vllm_omni/diffusion/layers/rope.py index 81e8c098f..461c25652 100644 --- a/vllm_omni/diffusion/layers/rope.py +++ b/vllm_omni/diffusion/layers/rope.py @@ -137,6 +137,14 @@ class RotaryEmbedding(CustomOp): else: return self.forward_native(x, cos, sin) + def forward_xpu( + self, + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + ) -> torch.Tensor: + return self.forward_native(x, cos, sin) + def forward_native( self, x: torch.Tensor, diff --git a/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py b/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py index 6bb282a80..9d216363d 100644 --- a/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py +++ b/vllm_omni/diffusion/models/longcat_image/longcat_image_transformer.py @@ -24,7 +24,7 @@ from vllm_omni.diffusion.distributed.parallel_state import ( get_sp_group, ) from vllm_omni.diffusion.forward_context import get_forward_context -from vllm_omni.utils.platform_utils import is_npu +from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) @@ -610,7 +610,7 @@ class LongCatImageTransformer2DModel(nn.Module): ids = torch.cat((txt_ids, img_ids), dim=0) - if is_npu(): + if current_omni_platform.is_npu(): freqs_cos, freqs_sin = self.pos_embed(ids.cpu()) image_rotary_emb = (freqs_cos.npu(), freqs_sin.npu()) else: diff --git a/vllm_omni/diffusion/worker/__init__.py b/vllm_omni/diffusion/worker/__init__.py index dfec4596b..8af028385 100644 --- a/vllm_omni/diffusion/worker/__init__.py +++ b/vllm_omni/diffusion/worker/__init__.py @@ -2,14 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Worker classes for diffusion models.""" -from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner -from vllm_omni.diffusion.worker.gpu_diffusion_worker import ( - GPUDiffusionWorker, +from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner +from vllm_omni.diffusion.worker.diffusion_worker import ( + DiffusionWorker, WorkerProc, ) __all__ = [ - "GPUDiffusionModelRunner", - "GPUDiffusionWorker", + "DiffusionModelRunner", + "DiffusionWorker", "WorkerProc", ] diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py similarity index 99% rename from vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py rename to vllm_omni/diffusion/worker/diffusion_model_runner.py index 0401655a8..7335162b0 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -34,7 +34,7 @@ from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec logger = init_logger(__name__) -class GPUDiffusionModelRunner: +class DiffusionModelRunner: """ Model runner that handles model loading and execution for diffusion models. diff --git a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py b/vllm_omni/diffusion/worker/diffusion_worker.py similarity index 94% rename from vllm_omni/diffusion/worker/gpu_diffusion_worker.py rename to vllm_omni/diffusion/worker/diffusion_worker.py index d8b962a58..0fad90a5b 100644 --- a/vllm_omni/diffusion/worker/gpu_diffusion_worker.py +++ b/vllm_omni/diffusion/worker/diffusion_worker.py @@ -4,7 +4,7 @@ Diffusion Worker for vLLM-Omni. Handles GPU infrastructure initialization and delegates model operations -to GPUDiffusionModelRunner. +to DiffusionModelRunner. """ import multiprocessing as mp @@ -31,13 +31,14 @@ from vllm_omni.diffusion.forward_context import set_forward_context from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager from vllm_omni.diffusion.profiler import CurrentProfiler from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner +from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner from vllm_omni.lora.request import LoRARequest +from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) -class GPUDiffusionWorker: +class DiffusionWorker: """ A worker that manages GPU infrastructure and delegates to the model runner. @@ -47,7 +48,7 @@ class GPUDiffusionWorker: - Memory management (sleep/wake) All model-related operations (loading, compilation, execution) are - delegated to GPUDiffusionModelRunner. + delegated to DiffusionModelRunner. """ def __init__( @@ -61,7 +62,7 @@ class GPUDiffusionWorker: self.od_config = od_config self.device: torch.device | None = None self.vllm_config: VllmConfig | None = None - self.model_runner: GPUDiffusionModelRunner | None = None + self.model_runner: DiffusionModelRunner | None = None self._sleep_saved_buffers: dict[str, torch.Tensor] = {} self.lora_manager: DiffusionLoRAManager | None = None self.init_device() @@ -79,8 +80,8 @@ class GPUDiffusionWorker: os.environ["WORLD_SIZE"] = str(world_size) # Setup device - self.device = torch.device(f"cuda:{rank}") - torch.cuda.set_device(self.device) + self.device = current_omni_platform.get_torch_device(rank) + current_omni_platform.set_device(self.device) # Create vllm_config for parallel configuration vllm_config = VllmConfig() @@ -105,7 +106,7 @@ class GPUDiffusionWorker: ) # Create model runner and load model - self.model_runner = GPUDiffusionModelRunner( + self.model_runner = DiffusionModelRunner( vllm_config=self.vllm_config, od_config=self.od_config, device=self.device, @@ -176,7 +177,7 @@ class GPUDiffusionWorker: """ from vllm.device_allocator.cumem import CuMemAllocator - free_bytes_before_sleep = torch.cuda.mem_get_info()[0] + free_bytes_before_sleep = current_omni_platform.get_free_memory() # Save the buffers before level 2 sleep if level == 2 and self.model_runner is not None: @@ -185,7 +186,9 @@ class GPUDiffusionWorker: allocator = CuMemAllocator.get_instance() allocator.sleep(offload_tags=("weights",) if level == 1 else tuple()) - free_bytes_after_sleep, total = torch.cuda.mem_get_info() + free_bytes_after_sleep = current_omni_platform.get_free_memory() + device_id = self.device.index if self.device.index is not None else 0 + total = current_omni_platform.get_device_total_memory(device_id) freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep used_bytes = total - free_bytes_after_sleep assert freed_bytes >= 0, "Memory usage increased after sleeping." @@ -270,9 +273,9 @@ class WorkerProc: self.gpu_id = gpu_id self._running = True - def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> GPUDiffusionWorker: + def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> DiffusionWorker: """Create a worker instance. Override in subclasses for different worker types.""" - return GPUDiffusionWorker( + return DiffusionWorker( local_rank=gpu_id, rank=gpu_id, od_config=od_config, @@ -379,6 +382,9 @@ class WorkerProc: broadcast_handle, ) -> None: """Worker initialization and execution loops.""" + from vllm_omni.plugins import load_omni_general_plugins + + load_omni_general_plugins() worker_proc = WorkerProc( od_config, gpu_id=rank, diff --git a/vllm_omni/diffusion/worker/npu/__init__.py b/vllm_omni/diffusion/worker/npu/__init__.py deleted file mode 100644 index b2210b5e2..000000000 --- a/vllm_omni/diffusion/worker/npu/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NPU worker classes for diffusion models.""" - -from vllm_omni.diffusion.worker.npu.npu_worker import NPUWorker, NPUWorkerProc - -__all__ = ["NPUWorker", "NPUWorkerProc"] diff --git a/vllm_omni/diffusion/worker/npu/npu_worker.py b/vllm_omni/diffusion/worker/npu/npu_worker.py deleted file mode 100644 index c4d2c60e5..000000000 --- a/vllm_omni/diffusion/worker/npu/npu_worker.py +++ /dev/null @@ -1,255 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import multiprocessing as mp -import os -import time -from collections.abc import Iterable -from contextlib import AbstractContextManager, nullcontext - -import torch -from vllm.config import LoadConfig, VllmConfig -from vllm.logger import init_logger -from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes - -from vllm_omni.diffusion.cache.selector import get_cache_backend -from vllm_omni.diffusion.data import ( - DiffusionOutput, - OmniDiffusionConfig, -) -from vllm_omni.diffusion.distributed.parallel_state import ( - destroy_distributed_env, - init_distributed_environment, - initialize_model_parallel, -) -from vllm_omni.diffusion.forward_context import set_forward_context -from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader -from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.diffusion.worker.gpu_diffusion_worker import WorkerProc - -logger = init_logger(__name__) - - -class NPUWorker: - """ - A worker that executes the model on a single NPU. - Inherits from GPUWorker and overrides device-specific initialization. - """ - - def __init__( - self, - local_rank: int, - rank: int, - od_config: OmniDiffusionConfig, - ): - self.local_rank = local_rank - self.rank = rank - self.od_config = od_config - self.pipeline = None - self.device = None - self._sleep_saved_buffers: dict[str, torch.Tensor] = {} - self.init_device_and_model() - - def init_device_and_model(self) -> None: - """Initialize the NPU device and load the model.""" - world_size = self.od_config.num_gpus - rank = self.rank - # Set environment variables for distributed initialization - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = str(self.od_config.master_port) - os.environ["LOCAL_RANK"] = str(self.local_rank) - os.environ["RANK"] = str(rank) - os.environ["WORLD_SIZE"] = str(world_size) - - device = torch.device(f"npu:{rank}") - torch.npu.set_device(device) - - # hack - vllm_config = VllmConfig() - vllm_config.parallel_config.tensor_parallel_size = self.od_config.parallel_config.tensor_parallel_size - vllm_config.parallel_config.data_parallel_size = self.od_config.parallel_config.data_parallel_size - self.vllm_config = vllm_config - with set_forward_context(vllm_config=vllm_config, omni_diffusion_config=self.od_config): - init_distributed_environment(world_size=world_size, rank=rank) - logger.info(f"Worker {self.rank}: Initialized device and distributed environment.") - parallel_config = self.od_config.parallel_config - initialize_model_parallel( - data_parallel_size=parallel_config.data_parallel_size, - cfg_parallel_size=parallel_config.cfg_parallel_size, - sequence_parallel_size=parallel_config.sequence_parallel_size, - ulysses_degree=parallel_config.ulysses_degree, - ring_degree=parallel_config.ring_degree, - tensor_parallel_size=parallel_config.tensor_parallel_size, - pipeline_parallel_size=parallel_config.pipeline_parallel_size, - ) - - load_config = LoadConfig() - model_loader = DiffusersPipelineLoader(load_config) - time_before_load = time.perf_counter() - with DeviceMemoryProfiler() as m: - self.pipeline = model_loader.load_model( - od_config=self.od_config, - load_device=f"npu:{rank}", - ) - time_after_load = time.perf_counter() - - logger.info( - "Model loading took %.4f GiB and %.6f seconds", - m.consumed_memory / GiB_bytes, - time_after_load - time_before_load, - ) - logger.info(f"Worker {self.rank}: Model loaded successfully.") - - # Setup cache backend based on type (both backends use enable()/reset() interface) - self.cache_backend = get_cache_backend(self.od_config.cache_backend, self.od_config.cache_config) - - if self.cache_backend is not None: - self.cache_backend.enable(self.pipeline) - - def generate(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput: - """ - Generate output for the given requests. - - Args: - requests: List of diffusion requests - - Returns: - DiffusionOutput with generated results - """ - return self.execute_model(requests, self.od_config) - - @torch.inference_mode() - def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput: - """ - Execute a forward pass. - """ - assert self.pipeline is not None - if not reqs or len(reqs) == 0: - raise ValueError("Cannot execute model with empty request list") - # TODO: dealing with first req for now - req = reqs[0] - - if req.generator is None and req.seed is not None: - req.generator = torch.Generator(device=self.device).manual_seed(req.seed) - - # Refresh cache context if needed - if ( - not getattr(req, "skip_cache_refresh", False) - and self.cache_backend is not None - and self.cache_backend.is_enabled() - ): - self.cache_backend.refresh(self.pipeline, req.num_inference_steps) - with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config): - output = self.pipeline.forward(req) - return output - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - return self.pipeline.load_weights(weights) - - def sleep(self, level: int = 1) -> bool: - """ - Put the worker to sleep. The worker should not process any requests. - The caller should guarantee that no requests are being processed - during the sleep period, before `wake_up` is called. - - Args: - level: The sleep level. Level 1 sleep will offload the model - weights and discard the kv cache. - Currently only support level 1. - """ - from vllm.device_allocator.cumem import CuMemAllocator - - free_bytes_before_sleep = torch.cuda.mem_get_info()[0] - - # Save the buffers before level 2 sleep - if level == 2: - model = self.pipeline - self._sleep_saved_buffers = {name: buffer.cpu().clone() for name, buffer in model.named_buffers()} - - allocator = CuMemAllocator.get_instance() - allocator.sleep(offload_tags=("weights",) if level == 1 else tuple()) - free_bytes_after_sleep, total = torch.cuda.mem_get_info() - freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep - used_bytes = total - free_bytes_after_sleep - assert freed_bytes >= 0, "Memory usage increased after sleeping." - logger.info( - "Sleep mode freed %.2f GiB memory, %.2f GiB memory is still in use.", - freed_bytes / GiB_bytes, - used_bytes / GiB_bytes, - ) - return True - - def wake_up(self, tags: list[str] | None = None) -> bool: - """ - Wake up the worker from sleep mode. See the sleep function - method for more details. - - Args: - tags: An optional list of tags to reallocate the worker memory - for specific memory allocations. Values must be in - `("weights")`. If None, all memory is reallocated. - wake_up should be called with all tags (or None) before the - worker is used again. - """ - from vllm.device_allocator.cumem import CuMemAllocator - - allocator = CuMemAllocator.get_instance() - allocator.wake_up(tags) - - # Restore the buffers after level 2 sleep - if len(self._sleep_saved_buffers): - model = self.pipeline - for name, buffer in model.named_buffers(): - if name in self._sleep_saved_buffers: - buffer.data.copy_(self._sleep_saved_buffers[name].data) - self._sleep_saved_buffers = {} - return True - - def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager: - if self.od_config.enable_sleep_mode: - from vllm.device_allocator.cumem import CuMemAllocator - - allocator = CuMemAllocator.get_instance() - if tag == "weights": - assert allocator.get_current_usage() == 0, "Sleep mode can only be used for one instance per process." - return allocator.use_memory_pool(tag=tag) - else: - return nullcontext() - - def shutdown(self) -> None: - destroy_distributed_env() - - -class NPUWorkerProc(WorkerProc): - """Wrapper that runs one NPUWorker in a separate process.""" - - def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> NPUWorker: - """Create an NPUWorker instead of GPUWorker.""" - return NPUWorker( - local_rank=gpu_id, - rank=gpu_id, - od_config=od_config, - ) - - @staticmethod - def worker_main( - rank: int, - od_config: OmniDiffusionConfig, - pipe_writer: mp.connection.Connection, - broadcast_handle, - ) -> None: - """Worker initialization and execution loops.""" - - worker_proc = NPUWorkerProc( - od_config, - gpu_id=rank, - broadcast_handle=broadcast_handle, - ) - logger.info(f"Worker {rank}: Scheduler loop started.") - pipe_writer.send( - { - "status": "ready", - "result_handle": worker_proc.result_mq_handle if rank == 0 else None, - } - ) - worker_proc.worker_busy_loop() - logger.info(f"Worker {rank}: Shutdown complete.") diff --git a/vllm_omni/engine/arg_utils.py b/vllm_omni/engine/arg_utils.py index cbe2001d7..21bb761d2 100644 --- a/vllm_omni/engine/arg_utils.py +++ b/vllm_omni/engine/arg_utils.py @@ -7,6 +7,7 @@ from vllm.transformers_utils.config import get_hf_text_config from vllm.v1.engine.async_llm import AsyncEngineArgs from vllm_omni.config import OmniModelConfig +from vllm_omni.plugins import load_omni_general_plugins logger = init_logger(__name__) @@ -89,7 +90,11 @@ class OmniEngineArgs(EngineArgs): f"Config attribute '{hf_config_name}' not found in hf_config, " "falling back to default get_hf_text_config" ) - return get_hf_text_config(hf_config) + return get_hf_text_config(hf_config) + + def __post_init__(self) -> None: + load_omni_general_plugins() + super().__post_init__() def _ensure_omni_models_registered(self): if hasattr(self, "_omni_models_registered"): @@ -187,7 +192,11 @@ class AsyncOmniEngineArgs(AsyncEngineArgs): f"Config attribute '{hf_config_name}' not found in hf_config, " "falling back to default get_hf_text_config" ) - return get_hf_text_config(hf_config) + return get_hf_text_config(hf_config) + + def __post_init__(self) -> None: + load_omni_general_plugins() + super().__post_init__() def _ensure_omni_models_registered(self): if hasattr(self, "_omni_models_registered"): diff --git a/vllm_omni/entrypoints/omni_stage.py b/vllm_omni/entrypoints/omni_stage.py index 0a5ee55be..2adb1c342 100644 --- a/vllm_omni/entrypoints/omni_stage.py +++ b/vllm_omni/entrypoints/omni_stage.py @@ -51,11 +51,27 @@ from vllm_omni.entrypoints.stage_utils import ( ) from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType, OmniSamplingParams, OmniTokensPrompt from vllm_omni.outputs import OmniRequestOutput -from vllm_omni.utils import detect_device_type logger = init_logger(__name__) +def _resolve_worker_cls(engine_args: dict[str, Any]) -> None: + worker_type = engine_args.pop("worker_type", None) + if not worker_type: + return + if engine_args.get("worker_cls"): + return + from vllm_omni.platforms import current_omni_platform + + worker_type = str(worker_type).lower() + if worker_type == "ar": + engine_args["worker_cls"] = current_omni_platform.get_omni_ar_worker_cls() + elif worker_type == "generation": + engine_args["worker_cls"] = current_omni_platform.get_omni_generation_worker_cls() + else: + raise ValueError(f"Unknown worker_type: {worker_type}") + + def _build_od_config(engine_args: dict[str, Any], model: str) -> dict[str, Any]: """Build OmniDiffusionConfig kwargs from engine args.""" od_config = engine_args.get("od_config", {}) @@ -483,6 +499,9 @@ def _stage_worker( import os as _os import time as _time + from vllm_omni.plugins import load_omni_general_plugins + + load_omni_general_plugins() # IMPORTANT: Ensure vLLM's internal multiprocessing workers (e.g., GPUARWorker / # GPUARModelRunner) are spawned with a fork-safe method. # Mooncake / gRPC / RDMA and CUDA/NCCL can deadlock under fork-with-threads. @@ -503,6 +522,9 @@ def _stage_worker( connectors_config = stage_payload.get("connectors_config", {}) stage_type: Literal["llm", "diffusion"] = stage_payload.get("stage_type", "llm") + if stage_type != "diffusion": + _resolve_worker_cls(engine_args) + # Aggregates for running average _agg_total_tokens = 0 _agg_total_gen_time_ms = 0.0 @@ -512,7 +534,9 @@ def _stage_worker( # Device mapping device_type = None try: - device_type = detect_device_type() + from vllm_omni.platforms import current_omni_platform + + device_type = current_omni_platform.device_type set_stage_devices(stage_id, runtime_cfg.get("devices"), device_type=device_type) except Exception as e: logger.warning("Device setup failed: %s", e) @@ -521,139 +545,137 @@ def _stage_worker( # when multiple instances start simultaneously # For TP/PP/DP/SP, we need to lock ALL devices that will be used by this stage lock_files = [] - if device_type == "cuda": - try: - import torch + try: + # Get all parallel sizes from engine_args or parallel_config (defaults to 1) + if "parallel_config" in engine_args: + parallel_config = engine_args["parallel_config"] + tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1) + pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1) + data_parallel_size = parallel_config.get("data_parallel_size", 1) + prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1) + sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1) + cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1) + else: + tensor_parallel_size = engine_args.get("tensor_parallel_size", 1) + pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1) + data_parallel_size = engine_args.get("data_parallel_size", 1) + prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1) + sequence_parallel_size = 1 # not use in omni model + cfg_parallel_size = 1 # not used in omni model - if torch.cuda.is_available(): - # Get all parallel sizes from engine_args or parallel_config (defaults to 1) - if "parallel_config" in engine_args: - parallel_config = engine_args["parallel_config"] - tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1) - pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1) - data_parallel_size = parallel_config.get("data_parallel_size", 1) - prefill_context_parallel_size = 1 # not used for diffusion - sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1) - cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1) - else: - tensor_parallel_size = engine_args.get("tensor_parallel_size", 1) - pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1) - data_parallel_size = engine_args.get("data_parallel_size", 1) - prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1) - sequence_parallel_size = 1 # not use in omni model - cfg_parallel_size = 1 # not used in omni model + # Calculate total number of devices needed for this stage + # For a single stage worker: + # - TP: splits model across devices (always needed) + # - PP: splits layers across pipeline stages, but each stage uses TP devices + # - DP: replicates model, but each replica uses TP devices + # - PCP: context parallelism, typically uses TP devices + # - SP: sequence parallelism, typically uses TP devices + # - CFG: Classifier-Free Guidance parallelism for diffusion models + # The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size + # (PP/DP/PCP are higher-level parallelism that don't add devices per stage) + num_devices_per_stage = ( + tensor_parallel_size + * pipeline_parallel_size + * data_parallel_size + * prefill_context_parallel_size + * sequence_parallel_size + * cfg_parallel_size + ) - # Calculate total number of devices needed for this stage - # For a single stage worker: - # - TP: splits model across GPUs (always needed) - # - PP: splits layers across pipelinestages, but each stage uses TP devices - # - DP: replicates model, but each replica uses TP devices - # - PCP: context parallelism, typically uses TP devices - # - SP: sequence parallelism, typically uses TP devices - # - CFG: Classifier-Free Guidance parallelism for diffusion models - # The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size - # (PP/DP/PCP are higher-level parallelism that don't add devices per stage) - num_devices_per_stage = ( - tensor_parallel_size - * pipeline_parallel_size - * data_parallel_size - * prefill_context_parallel_size - * sequence_parallel_size - * cfg_parallel_size - ) + # Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES) + # After set_stage_devices, this env var is set to physical device(s) + device_control_env = current_omni_platform.device_control_env_var + visible_devices_str = _os.environ.get(device_control_env) + physical_devices = [] - # Get physical device IDs from CUDA_VISIBLE_DEVICES - # After set_stage_devices, CUDA_VISIBLE_DEVICES is set to physical device(s) - cuda_visible_devices = _os.environ.get("CUDA_VISIBLE_DEVICES") - physical_devices = [] + if visible_devices_str: + try: + physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()] + except (ValueError, IndexError): + pass - if cuda_visible_devices: + if not physical_devices: + # Fallback: use logical device count if device control env var not set + num_devices = current_omni_platform.get_device_count() + physical_devices = list(range(num_devices)) + + # Determine which devices will be used (min of devices per stage and available devices) + num_devices_to_lock = min(num_devices_per_stage, len(physical_devices)) + devices_to_lock = physical_devices[:num_devices_to_lock] + + # Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order) + devices_to_lock = sorted(devices_to_lock) + + logger.debug( + "Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s", + tensor_parallel_size, + pipeline_parallel_size, + data_parallel_size, + prefill_context_parallel_size, + sequence_parallel_size, + cfg_parallel_size, + num_devices_to_lock, + devices_to_lock, + ) + + # Acquire exclusive locks for all devices using fcntl.flock + # Locks are automatically released when process dies + wait_start = _time.time() + acquired_lock_fds = [] # Store file descriptors to keep locks alive + + for device_id in devices_to_lock: + lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock" + lock_acquired = False + + while not lock_acquired: + try: + # Open or create the lock file + lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644) + + # Try to acquire exclusive lock (non-blocking first) try: - physical_devices = [int(x.strip()) for x in cuda_visible_devices.split(",") if x.strip()] - except (ValueError, IndexError): - pass + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + # Successfully acquired lock - write PID + _os.ftruncate(lock_fd, 0) # Clear file + _os.write(lock_fd, f"{_os.getpid()}\n".encode()) + _os.fsync(lock_fd) # Ensure written to disk + lock_acquired = True + acquired_lock_fds.append(lock_fd) + logger.debug("Acquired exclusive lock for device %s", device_id) + except BlockingIOError: + # Lock is held by another process + _os.close(lock_fd) - if not physical_devices: - # Fallback: use logical device count if CUDA_VISIBLE_DEVICES not set - num_devices = torch.cuda.device_count() - physical_devices = list(range(num_devices)) - - # Determine which devices will be used (min of devices per stage and available devices) - num_devices_to_lock = min(num_devices_per_stage, len(physical_devices)) - devices_to_lock = physical_devices[:num_devices_to_lock] - - # Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order) - devices_to_lock = sorted(devices_to_lock) - - logger.debug( - "Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d; will lock %d devices: %s", - tensor_parallel_size, - pipeline_parallel_size, - data_parallel_size, - prefill_context_parallel_size, - sequence_parallel_size, - num_devices_to_lock, - devices_to_lock, - ) - - # Acquire exclusive locks for all devices using fcntl.flock - # Locks are automatically released when process dies - wait_start = _time.time() - acquired_lock_fds = [] # Store file descriptors to keep locks alive - - for device_id in devices_to_lock: - lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock" - lock_acquired = False - - while not lock_acquired: - try: - # Open or create the lock file - lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644) - - # Try to acquire exclusive lock (non-blocking first) - try: - fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - # Successfully acquired lock - write PID - _os.ftruncate(lock_fd, 0) # Clear file - _os.write(lock_fd, f"{_os.getpid()}\n".encode()) - _os.fsync(lock_fd) # Ensure written to disk - lock_acquired = True - acquired_lock_fds.append(lock_fd) - logger.debug("Acquired exclusive lock for device %s", device_id) - except BlockingIOError: - # Lock is held by another process - _os.close(lock_fd) - - # Check if we've been waiting too long - if _time.time() - wait_start > stage_init_timeout: - logger.warning( - "Timeout waiting for device %s initialization lock, proceeding anyway", - device_id, - ) - break - - # Wait a bit before retrying - _time.sleep(0.1) - except OSError as e: - # Other error - log and continue without lock - logger.debug( - "Failed to acquire lock for device %s: %s, continuing anyway", + # Check if we've been waiting too long + if _time.time() - wait_start > stage_init_timeout: + logger.warning( + "Timeout waiting for device %s initialization lock, proceeding anyway", device_id, - e, ) - try: - _os.close(lock_fd) - except (OSError, NameError): - pass break - lock_files = acquired_lock_fds - except Exception as e: - logger.debug( - "[Stage-%s] Failed to set up sequential initialization lock: %s", - stage_id, - e, - ) + # Wait a bit before retrying + _time.sleep(0.1) + except OSError as e: + # Other error - log and continue without lock + logger.debug( + "Failed to acquire lock for device %s: %s, continuing anyway", + device_id, + e, + ) + try: + _os.close(lock_fd) + except (OSError, NameError): + pass + break + + lock_files = acquired_lock_fds + except Exception as e: + logger.debug( + "[Stage-%s] Failed to set up sequential initialization lock: %s", + stage_id, + e, + ) # Init engine based on stage_type logger.debug("[Stage-%s] Initializing %s engine with args keys=%s", stage_id, stage_type, list(engine_args.keys())) if engine_args.get("async_chunk", False): @@ -1001,6 +1023,9 @@ async def _stage_worker_async( import os as _os import time as _time + from vllm_omni.plugins import load_omni_general_plugins + + load_omni_general_plugins() # IMPORTANT: Ensure vLLM's internal multiprocessing workers (e.g., GPUARWorker / # GPUARModelRunner) are spawned with a fork-safe method. if _os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn": @@ -1018,6 +1043,9 @@ async def _stage_worker_async( connectors_config = stage_payload.get("connectors_config", {}) stage_type = stage_payload.get("stage_type", "llm") + if stage_type != "diffusion": + _resolve_worker_cls(engine_args) + in_q = omni_stage._in_q out_q = omni_stage._out_q @@ -1031,9 +1059,9 @@ async def _stage_worker_async( # Device mapping device_type = None try: - from vllm_omni.utils import detect_device_type + from vllm_omni.platforms import current_omni_platform - device_type = detect_device_type() + device_type = current_omni_platform.device_type set_stage_devices(stage_id, runtime_cfg.get("devices"), device_type=device_type) except Exception as e: logger.warning("Device setup failed: %s", e) @@ -1051,120 +1079,136 @@ async def _stage_worker_async( # Sequential initialization on the same device to avoid memory calculation errors # when multiple instances start simultaneously - # For TP, we need to lock ALL devices that will be used by this stage + # For TP/PP/DP/PCP, we need to lock ALL devices that will be used by this stage lock_files = [] - if device_type == "cuda": - try: - import torch + try: + # Get all parallel sizes from engine_args or parallel_config (defaults to 1) + if "parallel_config" in engine_args: + parallel_config = engine_args["parallel_config"] + tensor_parallel_size = parallel_config.get("tensor_parallel_size", 1) + pipeline_parallel_size = parallel_config.get("pipeline_parallel_size", 1) + data_parallel_size = parallel_config.get("data_parallel_size", 1) + prefill_context_parallel_size = parallel_config.get("prefill_context_parallel_size", 1) + sequence_parallel_size = parallel_config.get("sequence_parallel_size", 1) + cfg_parallel_size = parallel_config.get("cfg_parallel_size", 1) + else: + tensor_parallel_size = engine_args.get("tensor_parallel_size", 1) + pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1) + data_parallel_size = engine_args.get("data_parallel_size", 1) + prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1) + sequence_parallel_size = 1 # not use in omni model + cfg_parallel_size = 1 # not used in omni model - if torch.cuda.is_available(): - # Get all parallel sizes from engine_args (defaults to 1) - tensor_parallel_size = engine_args.get("tensor_parallel_size", 1) - pipeline_parallel_size = engine_args.get("pipeline_parallel_size", 1) - data_parallel_size = engine_args.get("data_parallel_size", 1) - prefill_context_parallel_size = engine_args.get("prefill_context_parallel_size", 1) + # Calculate total number of devices needed for this stage + # For a single stage worker: + # - TP: splits model across devices (always needed) + # - PP: splits layers across pipeline stages, but each stage uses TP devices + # - DP: replicates model, but each replica uses TP devices + # - PCP: context parallelism, typically uses TP devices + # - SP: sequence parallelism, typically uses TP devices + # - CFG: Classifier-Free Guidance parallelism for diffusion models + # The number of devices per stage is determined by TP * PP * DP * PCP * SP * CFG size + # (PP/DP/PCP are higher-level parallelism that don't add devices per stage) + num_devices_per_stage = ( + tensor_parallel_size + * pipeline_parallel_size + * data_parallel_size + * prefill_context_parallel_size + * sequence_parallel_size + * cfg_parallel_size + ) - # Calculate total number of devices needed for this stage - # For a single stage worker in omni: - # - TP: splits model across GPUs (always needed) - # - PP: splits layers across stages, but each stage uses TP devices - # - DP: replicates model, but each replica uses TP devices - # - PCP: context parallelism, typically uses TP devices - # The number of devices per stage is determined by TP * PP * DP * PCP size - # (PP/DP/PCP are higher-level parallelism that don't add devices per stage) - num_devices_per_stage = ( - tensor_parallel_size * pipeline_parallel_size * data_parallel_size * prefill_context_parallel_size - ) + # Get physical device IDs from device control env var (e.g., CUDA_VISIBLE_DEVICES) + # After set_stage_devices, this env var is set to physical device(s) + device_control_env = current_omni_platform.device_control_env_var + visible_devices_str = _os.environ.get(device_control_env) + physical_devices = [] - # Get physical device IDs from CUDA_VISIBLE_DEVICES - # After set_stage_devices, CUDA_VISIBLE_DEVICES is set to physical device(s) - cuda_visible_devices = _os.environ.get("CUDA_VISIBLE_DEVICES") - physical_devices = [] + if visible_devices_str: + try: + physical_devices = [int(x.strip()) for x in visible_devices_str.split(",") if x.strip()] + except (ValueError, IndexError): + pass - if cuda_visible_devices: + if not physical_devices: + # Fallback: use logical device count if device control env var not set + num_devices = current_omni_platform.get_device_count() + physical_devices = list(range(num_devices)) + + # Determine which devices will be used (min of devices per stage and available devices) + num_devices_to_lock = min(num_devices_per_stage, len(physical_devices)) + devices_to_lock = physical_devices[:num_devices_to_lock] + + # Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order) + devices_to_lock = sorted(devices_to_lock) + + logger.debug( + "Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d, SP=%d, CFG=%d; will lock %d devices: %s", + tensor_parallel_size, + pipeline_parallel_size, + data_parallel_size, + prefill_context_parallel_size, + sequence_parallel_size, + cfg_parallel_size, + num_devices_to_lock, + devices_to_lock, + ) + + # Acquire exclusive locks for all devices using fcntl.flock + # Locks are automatically released when process dies + wait_start = _time.time() + acquired_lock_fds = [] # Store file descriptors to keep locks alive + + for device_id in devices_to_lock: + lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock" + lock_acquired = False + + while not lock_acquired: + try: + # Open or create the lock file + lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644) + + # Try to acquire exclusive lock (non-blocking first) try: - physical_devices = [int(x.strip()) for x in cuda_visible_devices.split(",") if x.strip()] - except (ValueError, IndexError): - pass + fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + # Successfully acquired lock - write PID + _os.ftruncate(lock_fd, 0) # Clear file + _os.write(lock_fd, f"{_os.getpid()}\n".encode()) + _os.fsync(lock_fd) # Ensure written to disk + lock_acquired = True + acquired_lock_fds.append(lock_fd) + logger.debug("Acquired exclusive lock for device %s", device_id) + except BlockingIOError: + # Lock is held by another process + _os.close(lock_fd) - if not physical_devices: - # Fallback: use logical device count if CUDA_VISIBLE_DEVICES not set - num_devices = torch.cuda.device_count() - physical_devices = list(range(num_devices)) - - # Determine which devices will be used (min of devices per stage and available devices) - num_devices_to_lock = min(num_devices_per_stage, len(physical_devices)) - devices_to_lock = physical_devices[:num_devices_to_lock] - - # Sort devices_to_lock to prevent deadlock (all processes acquire locks in same order) - devices_to_lock = sorted(devices_to_lock) - - logger.debug( - "Parallel config: TP=%d, PP=%d, DP=%d, PCP=%d; will lock %d devices: %s", - tensor_parallel_size, - pipeline_parallel_size, - data_parallel_size, - prefill_context_parallel_size, - num_devices_to_lock, - devices_to_lock, - ) - - # Acquire exclusive locks for all devices using fcntl.flock - # Locks are automatically released when process dies - wait_start = _time.time() - acquired_lock_fds = [] # Store file descriptors to keep locks alive - - for device_id in devices_to_lock: - lock_file = f"/tmp/vllm_omni_device_{device_id}_init.lock" - lock_acquired = False - - while not lock_acquired: - try: - # Open or create the lock file - lock_fd = _os.open(lock_file, _os.O_CREAT | _os.O_RDWR, 0o644) - - # Try to acquire exclusive lock (non-blocking first) - try: - fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - # Successfully acquired lock - write PID - _os.ftruncate(lock_fd, 0) # Clear file - _os.write(lock_fd, f"{_os.getpid()}\n".encode()) - _os.fsync(lock_fd) # Ensure written to disk - lock_acquired = True - acquired_lock_fds.append(lock_fd) - logger.debug("Acquired exclusive lock for device %s", device_id) - except BlockingIOError: - # Lock is held by another process - _os.close(lock_fd) - - # Check if we've been waiting too long - if _time.time() - wait_start > stage_init_timeout: - logger.warning( - "Timeout waiting for device %s initialization lock, " - "proceeding anyway with timeout %s", - device_id, - stage_init_timeout, - ) - break - - # Wait a bit before retrying - _time.sleep(0.1) - except OSError as e: - # Other error - log and continue without lock - logger.debug( - "Failed to acquire lock for device %s: %s, continuing anyway", + # Check if we've been waiting too long + if _time.time() - wait_start > stage_init_timeout: + logger.warning( + "Timeout waiting for device %s initialization lock, proceeding anyway with timeout %s", device_id, - e, + stage_init_timeout, ) - try: - _os.close(lock_fd) - except (OSError, NameError): - pass break - lock_files = acquired_lock_fds - except Exception as e: - logger.debug("Failed to set up sequential initialization lock: %s", e) + # Wait a bit before retrying + _time.sleep(0.1) + except OSError as e: + # Other error - log and continue without lock + logger.debug( + "Failed to acquire lock for device %s: %s, continuing anyway", + device_id, + e, + ) + try: + _os.close(lock_fd) + except (OSError, NameError): + pass + break + + lock_files = acquired_lock_fds + except Exception as e: + logger.debug("Failed to set up sequential initialization lock: %s", e) # Init engine based on stage_type logger.debug( diff --git a/vllm_omni/entrypoints/stage_utils.py b/vllm_omni/entrypoints/stage_utils.py index da837ce9a..74ad42f04 100644 --- a/vllm_omni/entrypoints/stage_utils.py +++ b/vllm_omni/entrypoints/stage_utils.py @@ -57,12 +57,12 @@ def set_stage_devices( - CUDA: Sets CUDA_VISIBLE_DEVICES and calls torch.cuda.set_device() - NPU: Sets ASCEND_RT_VISIBLE_DEVICES and calls torch.npu.set_device() """ - from vllm_omni.utils import detect_device_type, get_device_control_env_var + from vllm_omni.platforms import current_omni_platform if device_type is None: - device_type = detect_device_type() + device_type = current_omni_platform.device_type - env_var = get_device_control_env_var() + env_var = current_omni_platform.device_control_env_var try: selected_physical: int | None = None diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index f1ed4d843..4ebff3290 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -10,7 +10,7 @@ from vllm.transformers_utils.config import get_config, get_hf_file_to_dict from vllm.transformers_utils.repo_utils import file_or_path_exists from vllm_omni.entrypoints.stage_utils import _to_dict -from vllm_omni.utils import detect_device_type, is_rocm +from vllm_omni.platforms import current_omni_platform # Get the project root directory (2 levels up from this file) PROJECT_ROOT = Path(__file__).parent.parent.parent @@ -164,16 +164,12 @@ def resolve_model_config_path(model: str) -> str: f"Model is not in standard transformers format and does not have model_index.json. " f"Please ensure the model has proper configuration files with 'model_type' field" ) - device_type = detect_device_type() - # Try device-specific config first - if device_type != "cuda" or is_rocm(): - device_config_file = f"vllm_omni/model_executor/stage_configs/{device_type}/{model_type}.yaml" - if is_rocm(): - device_config_file = f"vllm_omni/model_executor/stage_configs/rocm/{model_type}.yaml" - device_config_path = PROJECT_ROOT / device_config_file - if os.path.exists(device_config_path): - return str(device_config_path) + default_config_path = current_omni_platform.get_default_stage_config_path() + model_type_str = f"{model_type}.yaml" + complete_config_path = PROJECT_ROOT / default_config_path / model_type_str + if os.path.exists(complete_config_path): + return str(complete_config_path) # Fall back to default config stage_config_file = f"vllm_omni/model_executor/stage_configs/{model_type}.yaml" diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py index 45a1447b3..f4afe675a 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni.py @@ -36,7 +36,7 @@ from vllm_omni.model_executor.model_loader.weight_utils import download_weights_ from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges from vllm_omni.model_executor.models.vision import get_llm_pos_ids_for_vision -from vllm_omni.utils.platform_utils import is_npu +from vllm_omni.platforms import current_omni_platform TALKER_CODEC_EOS_TOKEN_ID = 8294 TALKER_CODEC_BOS_TOKEN_ID = 8293 @@ -247,7 +247,7 @@ class Qwen2_5OmniForConditionalGeneration( if inputs_embeds is not None and inputs_embeds.device != thinker_dev: inputs_embeds = inputs_embeds.to(thinker_dev) - if is_npu(): + if current_omni_platform.is_npu(): # TODO: remove this hack when NPU supports batched inputs properly thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids # For MRoPE, positions shape is [3, num_tokens] (T/H/W), don't slice it diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py index c60015c0e..9d176ac6c 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py @@ -35,7 +35,7 @@ from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.sample.sampler import Sampler from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import cap_and_align_mel_length, resolve_max_mel_frames -from vllm_omni.utils.platform_utils import is_npu +from vllm_omni.platforms import current_omni_platform # Provide a no-op auto_docstring decorator to satisfy annotations if missing @@ -732,7 +732,7 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) -> beta = 0.0 # TODO: When torch.kaiser_window supports NPU, remove the device="cpu" argument - if is_npu(): + if current_omni_platform.is_npu(): kaiser_window = torch.kaiser_window( kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu" ).to("npu") @@ -796,7 +796,7 @@ class UpSample1d(nn.Module): def forward(self, hidden_states): channels = hidden_states.shape[1] - if is_npu(): + if current_omni_platform.is_npu(): # TODO: When F.pad supports replicate mode on NPU, remove this branch input_dtype = hidden_states.dtype # F.pad in NPU doesn't support BF16 when mode is replicate. @@ -843,7 +843,7 @@ class DownSample1d(nn.Module): def forward(self, hidden_states): channels = hidden_states.shape[1] - if is_npu(): + if current_omni_platform.is_npu(): input_dtype = hidden_states.dtype # F.pad in NPU doesn't support BF16 when mode is replicate. # To ensure the accuracy, manually pad the input tensor. diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py index a42bd8e76..37753ee47 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py @@ -35,7 +35,7 @@ from vllm.v1.sample.sampler import Sampler from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin from vllm_omni.model_executor.models.output_templates import OmniOutput from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, safe_tensor_reshape -from vllm_omni.utils.platform_utils import is_npu +from vllm_omni.platforms import current_omni_platform # Special token IDs for Qwen3 Omni MoE # Reference: https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json @@ -269,7 +269,7 @@ class Qwen3OmniMoeForConditionalGeneration( # ========== Stage 1: Thinker ========== if self.model_stage == "thinker": thinker_dev = self._module_device(self.thinker) - if is_npu(): + if current_omni_platform.is_npu(): # Normalize to batched inputs if needed _added_batch_dim = False if input_ids is not None and input_ids.ndim == 1: @@ -308,7 +308,7 @@ class Qwen3OmniMoeForConditionalGeneration( "capture_layer_indices": [0, int(accept_layer)], "return_hidden_states": True, } - if is_npu(): + if current_omni_platform.is_npu(): # TODO: remove this hack when NPU supports batched inputs properly thinker_input_ids = input_ids[0] if input_ids is not None and _added_batch_dim else input_ids thinker_inputs_embeds = ( diff --git a/vllm_omni/model_executor/stage_configs/bagel.yaml b/vllm_omni/model_executor/stage_configs/bagel.yaml index 96bdc01b6..93d2b611a 100644 --- a/vllm_omni/model_executor/stage_configs/bagel.yaml +++ b/vllm_omni/model_executor/stage_configs/bagel.yaml @@ -9,7 +9,7 @@ stage_args: engine_args: model_stage: thinker model_arch: BagelForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.4 enforce_eager: true diff --git a/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml b/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml index 678d5775f..86c0330a6 100644 --- a/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml +++ b/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml @@ -9,7 +9,7 @@ stage_args: engine_args: model_stage: thinker model_arch: BagelForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.4 enforce_eager: true diff --git a/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml b/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml index e3d740ad5..e6ed97660 100644 --- a/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml @@ -11,7 +11,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true # Now we only support eager mode @@ -40,7 +40,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true @@ -69,7 +69,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml b/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml index aae194d37..68f5817a7 100644 --- a/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen2_5_omni_multiconnector.yaml @@ -11,7 +11,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true # Now we only support eager mode @@ -41,7 +41,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true @@ -73,7 +73,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.3 enforce_eager: true diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml index 9c422d76e..d0380a39d 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml @@ -14,7 +14,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: true @@ -45,7 +45,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: true @@ -76,7 +76,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml index 55d455f56..42748960e 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml @@ -14,7 +14,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: false @@ -46,7 +46,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: false @@ -77,7 +77,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml index 4abd3835a..69102df6f 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml @@ -13,7 +13,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: true @@ -46,7 +46,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: true @@ -82,7 +82,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml b/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml index 40799fe7f..d408dbab9 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml @@ -7,7 +7,7 @@ stage_args: engine_args: model_stage: qwen3_tts model_arch: Qwen3TTSForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/platforms/__init__.py b/vllm_omni/platforms/__init__.py new file mode 100644 index 000000000..b76c59a67 --- /dev/null +++ b/vllm_omni/platforms/__init__.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import logging +import traceback +from itertools import chain +from typing import TYPE_CHECKING + +from vllm.utils.import_utils import resolve_obj_by_qualname +from vllm.utils.torch_utils import supports_xccl + +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum +from vllm_omni.plugins import ( + OMNI_PLATFORM_PLUGINS_GROUP, + load_omni_plugins_by_group, +) + +logger = logging.getLogger(__name__) + + +def cuda_omni_platform_plugin() -> str | None: + """Check if CUDA OmniPlatform should be activated.""" + is_cuda = False + logger.debug("Checking if CUDA OmniPlatform is available.") + try: + from vllm.utils.import_utils import import_pynvml + + pynvml = import_pynvml() + pynvml.nvmlInit() + try: + if pynvml.nvmlDeviceGetCount() > 0: + is_cuda = True + logger.debug("Confirmed CUDA OmniPlatform is available.") + else: + logger.debug("CUDA OmniPlatform is not available because no GPU is found.") + finally: + pynvml.nvmlShutdown() + except Exception as e: + logger.debug("CUDA OmniPlatform is not available because: %s", str(e)) + + return "vllm_omni.platforms.cuda.platform.CudaOmniPlatform" if is_cuda else None + + +def rocm_omni_platform_plugin() -> str | None: + """Check if ROCm OmniPlatform should be activated.""" + is_rocm = False + logger.debug("Checking if ROCm OmniPlatform is available.") + try: + import amdsmi + + amdsmi.amdsmi_init() + try: + if len(amdsmi.amdsmi_get_processor_handles()) > 0: + is_rocm = True + logger.debug("Confirmed ROCm OmniPlatform is available.") + else: + logger.debug("ROCm OmniPlatform is not available because no GPU is found.") + finally: + amdsmi.amdsmi_shut_down() + except Exception as e: + logger.debug("ROCm OmniPlatform is not available because: %s", str(e)) + + return "vllm_omni.platforms.rocm.platform.RocmOmniPlatform" if is_rocm else None + + +def npu_omni_platform_plugin() -> str | None: + """Check if NPU OmniPlatform should be activated.""" + is_npu = False + logger.debug("Checking if NPU OmniPlatform is available.") + try: + import torch + + if hasattr(torch, "npu") and torch.npu.is_available(): + is_npu = True + logger.debug("Confirmed NPU OmniPlatform is available.") + except Exception as e: + logger.debug("NPU OmniPlatform is not available because: %s", str(e)) + + return "vllm_omni.platforms.npu.platform.NPUOmniPlatform" if is_npu else None + + +def xpu_omni_platform_plugin() -> str | None: + """Check if XPU OmniPlatform should be activated.""" + is_xpu = False + logger.debug("Checking if XPU OmniPlatform is available.") + try: + # installed IPEX if the machine has XPUs. + import intel_extension_for_pytorch # noqa: F401 + import torch + + if supports_xccl(): + dist_backend = "xccl" + else: + dist_backend = "ccl" + import oneccl_bindings_for_pytorch # noqa: F401 + + if hasattr(torch, "xpu") and torch.xpu.is_available(): + is_xpu = True + from vllm_omni.platforms.xpu import XPUOmniPlatform + + XPUOmniPlatform.dist_backend = dist_backend + logger.debug("Confirmed %s backend is available.", XPUOmniPlatform.dist_backend) + logger.debug("Confirmed XPU platform is available.") + except Exception as e: + logger.debug("XPU omni platform is not available because: %s", str(e)) + + return "vllm_omni.platforms.xpu.platform.XPUOmniPlatform" if is_xpu else None + + +builtin_omni_platform_plugins = { + "cuda": cuda_omni_platform_plugin, + "rocm": rocm_omni_platform_plugin, + "npu": npu_omni_platform_plugin, + "xpu": xpu_omni_platform_plugin, +} + + +def resolve_current_omni_platform_cls_qualname() -> str: + """Resolve the current OmniPlatform class qualified name.""" + platform_plugins = load_omni_plugins_by_group(OMNI_PLATFORM_PLUGINS_GROUP) + + activated_plugins = [] + + for name, func in chain(builtin_omni_platform_plugins.items(), platform_plugins.items()): + try: + assert callable(func) + platform_cls_qualname = func() + if platform_cls_qualname is not None: + activated_plugins.append(name) + except Exception: + pass + + activated_builtin_plugins = list(set(activated_plugins) & set(builtin_omni_platform_plugins.keys())) + activated_oot_plugins = list(set(activated_plugins) & set(platform_plugins.keys())) + + if len(activated_oot_plugins) >= 2: + raise RuntimeError(f"Only one OmniPlatform plugin can be activated, but got: {activated_oot_plugins}") + elif len(activated_oot_plugins) == 1: + platform_cls_qualname = platform_plugins[activated_oot_plugins[0]]() + logger.info("OmniPlatform plugin %s is activated", activated_oot_plugins[0]) + elif len(activated_builtin_plugins) >= 2: + raise RuntimeError(f"Only one OmniPlatform plugin can be activated, but got: {activated_builtin_plugins}") + elif len(activated_builtin_plugins) == 1: + platform_cls_qualname = builtin_omni_platform_plugins[activated_builtin_plugins[0]]() + logger.debug("Automatically detected OmniPlatform %s.", activated_builtin_plugins[0]) + else: + platform_cls_qualname = "vllm_omni.platforms.interface.UnspecifiedOmniPlatform" + logger.debug("No platform detected, vLLM-Omni is running on UnspecifiedOmniPlatform") + + return platform_cls_qualname + + +_current_omni_platform = None +_init_trace: str = "" + +if TYPE_CHECKING: + current_omni_platform: OmniPlatform + + +def __getattr__(name: str): + if name == "current_omni_platform": + # Lazy init current_omni_platform + global _current_omni_platform + if _current_omni_platform is None: + platform_cls_qualname = resolve_current_omni_platform_cls_qualname() + _current_omni_platform = resolve_obj_by_qualname(platform_cls_qualname)() + global _init_trace + _init_trace = "".join(traceback.format_stack()) + return _current_omni_platform + elif name in globals(): + return globals()[name] + else: + raise AttributeError(f"No attribute named '{name}' exists in {__name__}.") + + +def __setattr__(name: str, value): # noqa: N807 + if name == "current_omni_platform": + global _current_omni_platform + _current_omni_platform = value + elif name in globals(): + globals()[name] = value + else: + raise AttributeError(f"No attribute named '{name}' exists in {__name__}.") + + +__all__ = [ + "OmniPlatform", + "OmniPlatformEnum", + "current_omni_platform", + "_init_trace", +] diff --git a/vllm_omni/platforms/cuda/__init__.py b/vllm_omni/platforms/cuda/__init__.py new file mode 100644 index 000000000..91ef864be --- /dev/null +++ b/vllm_omni/platforms/cuda/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.platforms.cuda.platform import CudaOmniPlatform + +__all__ = ["CudaOmniPlatform"] diff --git a/vllm_omni/platforms/cuda/platform.py b/vllm_omni/platforms/cuda/platform.py new file mode 100644 index 000000000..9c65877be --- /dev/null +++ b/vllm_omni/platforms/cuda/platform.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from vllm.logger import init_logger +from vllm.platforms.cuda import CudaPlatformBase +from vllm.platforms.interface import DeviceCapability + +from vllm_omni.diffusion.attention.backends.registry import DiffusionAttentionBackendEnum +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum + +logger = init_logger(__name__) + + +class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): + """CUDA/GPU implementation of OmniPlatform (default). + + Inherits all CUDA-specific implementations from vLLM's CudaPlatform, + and adds Omni-specific interfaces from OmniPlatform. + """ + + _omni_enum = OmniPlatformEnum.CUDA + + @classmethod + def get_omni_ar_worker_cls(cls) -> str: + return "vllm_omni.worker.gpu_ar_worker.GPUARWorker" + + @classmethod + def get_omni_generation_worker_cls(cls) -> str: + return "vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker" + + @classmethod + def get_default_stage_config_path(cls) -> str: + return "vllm_omni/model_executor/stage_configs" + + @classmethod + def get_diffusion_attn_backend_cls( + cls, + selected_backend: str | None, + head_size: int, + ) -> str: + # Check compute capability for Flash Attention support + # Flash Attention requires compute capability >= 8.0 and < 10.0 + compute_capability = cls.get_device_capability() + flash_attn_supported = False + if compute_capability is not None: + major, minor = compute_capability + capability = major * 10 + minor + flash_attn_supported = 80 <= capability < 100 + + if selected_backend is not None: + backend_upper = selected_backend.upper() + if backend_upper == "FLASH_ATTN" and not flash_attn_supported: + logger.warning( + "Flash Attention requires GPU with compute capability >= 8.0 " + "and < 10.0. Falling back to TORCH_SDPA backend." + ) + logger.info("Defaulting to diffusion attention backend SDPA") + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + backend = DiffusionAttentionBackendEnum[backend_upper] + logger.info("Using diffusion attention backend '%s'", backend_upper) + return backend.get_path() + + if flash_attn_supported: + logger.info("Defaulting to diffusion attention backend FLASH_ATTN") + return DiffusionAttentionBackendEnum.FLASH_ATTN.get_path() + + logger.info("Defaulting to diffusion attention backend SDPA") + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + + @classmethod + def get_torch_device(cls, local_rank: int | None = None) -> torch.device: + if local_rank is None: + return torch.device("cuda") + return torch.device("cuda", local_rank) + + @classmethod + def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: + major, minor = torch.cuda.get_device_capability(device_id) + return DeviceCapability(major=major, minor=minor) + + @classmethod + def get_device_count(cls) -> int: + return torch.cuda.device_count() + + @classmethod + def get_device_version(cls) -> str | None: + return torch.version.cuda + + @classmethod + def synchronize(cls) -> None: + torch.cuda.synchronize() + + @classmethod + def get_free_memory(cls, device: torch.device | None = None) -> int: + free, _ = torch.cuda.mem_get_info(device) + return free + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + return torch.cuda.get_device_name(device_id) diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py new file mode 100644 index 000000000..2aac4b7a1 --- /dev/null +++ b/vllm_omni/platforms/interface.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from enum import Enum + +import torch +from vllm.platforms import Platform + + +class OmniPlatformEnum(Enum): + """Enum for supported Omni platforms.""" + + CUDA = "cuda" + ROCM = "rocm" + NPU = "npu" + XPU = "xpu" + UNSPECIFIED = "unspecified" + + +class OmniPlatform(Platform): + """ + Abstract base class for vllm-omni Platform. + + Inherits from vLLM's Platform and adds Omni-specific interfaces. + This gives OmniPlatform all vLLM Platform capabilities plus + Omni-specific methods. + """ + + _omni_enum: OmniPlatformEnum + + def is_npu(self) -> bool: + return self._omni_enum == OmniPlatformEnum.NPU + + @classmethod + def get_omni_ar_worker_cls(cls) -> str: + raise NotImplementedError + + @classmethod + def get_omni_generation_worker_cls(cls) -> str: + raise NotImplementedError + + @classmethod + def get_default_stage_config_path(cls) -> str: + raise NotImplementedError + + @classmethod + def get_diffusion_attn_backend_cls( + cls, + selected_backend: str | None, + head_size: int, + ) -> str: + """Get the diffusion attention backend class path for this platform. + + This method selects the appropriate attention backend for diffusion + models based on platform capabilities and user preferences. + + Args: + selected_backend: User-selected backend name (e.g., "FLASH_ATTN", + "TORCH_SDPA", "SAGE_ATTN"). If None, uses platform default. + head_size: Attention head size. + + Returns: + Fully qualified class path of the selected backend. + """ + raise NotImplementedError + + @classmethod + def get_torch_device(cls, local_rank: int | None = None) -> torch.device: + raise NotImplementedError + + @classmethod + def get_device_count(cls) -> int: + raise NotImplementedError + + @classmethod + def get_device_version(cls) -> str | None: + raise NotImplementedError + + @classmethod + def synchronize(cls) -> None: + raise NotImplementedError + + @classmethod + def get_free_memory(cls, device: torch.device | None = None) -> int: + raise NotImplementedError + + +class UnspecifiedOmniPlatform(OmniPlatform): + _omni_enum = OmniPlatformEnum.UNSPECIFIED + device_type = "" diff --git a/vllm_omni/platforms/npu/__init__.py b/vllm_omni/platforms/npu/__init__.py new file mode 100644 index 000000000..fb95f7b29 --- /dev/null +++ b/vllm_omni/platforms/npu/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.platforms.npu.platform import NPUOmniPlatform + +__all__ = ["NPUOmniPlatform"] diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py new file mode 100644 index 000000000..429297bbd --- /dev/null +++ b/vllm_omni/platforms/npu/platform.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from vllm.logger import init_logger +from vllm_ascend.platform import NPUPlatform + +from vllm_omni.diffusion.attention.backends.registry import DiffusionAttentionBackendEnum +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum + +logger = init_logger(__name__) + + +class NPUOmniPlatform(OmniPlatform, NPUPlatform): + """NPU/Ascend implementation of OmniPlatform. + + Inherits all NPU-specific implementations from vllm-ascend's NPUPlatform, + and adds Omni-specific interfaces from OmniPlatform. + """ + + _omni_enum = OmniPlatformEnum.NPU + dist_backend: str = "hccl" + + @classmethod + def get_omni_ar_worker_cls(cls) -> str: + return "vllm_omni.platforms.npu.worker.npu_ar_worker.NPUARWorker" + + @classmethod + def get_omni_generation_worker_cls(cls) -> str: + return "vllm_omni.platforms.npu.worker.npu_generation_worker.NPUGenerationWorker" + + @classmethod + def get_default_stage_config_path(cls) -> str: + return "vllm_omni/platforms/npu/stage_configs" + + @classmethod + def get_diffusion_attn_backend_cls( + cls, + selected_backend: str | None, + head_size: int, + ) -> str: + from importlib.util import find_spec + + if selected_backend is not None: + backend_upper = selected_backend.upper() + backend = DiffusionAttentionBackendEnum[backend_upper] + logger.info("Using diffusion attention backend '%s'", backend_upper) + return backend.get_path() + + # Try FLASH_ATTN if mindiesd is available, otherwise fall back to SDPA + if find_spec("mindiesd"): + logger.info("Defaulting to diffusion attention backend FLASH_ATTN") + return DiffusionAttentionBackendEnum.FLASH_ATTN.get_path() + + logger.info("Falling back to diffusion attention backend SDPA") + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + + @classmethod + def get_torch_device(cls, local_rank: int | None = None) -> torch.device: + if local_rank is None: + return torch.device("npu") + return torch.device("npu", local_rank) + + @classmethod + def get_device_count(cls) -> int: + return torch.npu.device_count() + + @classmethod + def get_device_version(cls) -> str | None: + return None + + @classmethod + def synchronize(cls) -> None: + torch.npu.synchronize() + + @classmethod + def get_free_memory(cls, device: torch.device | None = None) -> int: + free, _ = torch.npu.mem_get_info(device) + return free + + @classmethod + def get_device_total_memory(cls, device_id: int = 0) -> int: + device_props = torch.npu.get_device_properties(device_id) + return device_props.total_memory diff --git a/vllm_omni/model_executor/stage_configs/npu/qwen2_5_omni.yaml b/vllm_omni/platforms/npu/stage_configs/qwen2_5_omni.yaml similarity index 89% rename from vllm_omni/model_executor/stage_configs/npu/qwen2_5_omni.yaml rename to vllm_omni/platforms/npu/stage_configs/qwen2_5_omni.yaml index 49085276f..a02d84ed6 100644 --- a/vllm_omni/model_executor/stage_configs/npu/qwen2_5_omni.yaml +++ b/vllm_omni/platforms/npu/stage_configs/qwen2_5_omni.yaml @@ -9,10 +9,10 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 - enforce_eager: true # Now we only support eager mode + enforce_eager: false trust_remote_code: true engine_output_type: latent enable_prefix_caching: false @@ -36,10 +36,10 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 - enforce_eager: true + enforce_eager: false trust_remote_code: true enable_prefix_caching: false engine_output_type: latent @@ -58,12 +58,12 @@ stage_args: stage_type: llm # Use llm stage type to launch OmniLLM runtime: process: true - devices: "0" # Example: use a different NPU than the previous stage; use "0" if single NPU + devices: "2" # Example: use a different NPU than the previous stage; use "0" if single NPU max_batch_size: 1 engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/vllm_omni/model_executor/stage_configs/npu/qwen3_omni_moe.yaml b/vllm_omni/platforms/npu/stage_configs/qwen3_omni_moe.yaml similarity index 93% rename from vllm_omni/model_executor/stage_configs/npu/qwen3_omni_moe.yaml rename to vllm_omni/platforms/npu/stage_configs/qwen3_omni_moe.yaml index eadfea570..fdf817239 100644 --- a/vllm_omni/model_executor/stage_configs/npu/qwen3_omni_moe.yaml +++ b/vllm_omni/platforms/npu/stage_configs/qwen3_omni_moe.yaml @@ -12,7 +12,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: false @@ -41,7 +41,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_ar_worker.NPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.2 enforce_eager: false @@ -71,7 +71,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/model_executor/stage_configs/npu/qwen3_tts.yaml b/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml similarity index 89% rename from vllm_omni/model_executor/stage_configs/npu/qwen3_tts.yaml rename to vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml index 86b5ca60e..d408dbab9 100644 --- a/vllm_omni/model_executor/stage_configs/npu/qwen3_tts.yaml +++ b/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml @@ -7,7 +7,7 @@ stage_args: engine_args: model_stage: qwen3_tts model_arch: Qwen3TTSForConditionalGeneration - worker_cls: vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/worker/npu/__init__.py b/vllm_omni/platforms/npu/worker/__init__.py similarity index 100% rename from vllm_omni/worker/npu/__init__.py rename to vllm_omni/platforms/npu/worker/__init__.py diff --git a/vllm_omni/worker/npu/npu_ar_model_runner.py b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py similarity index 99% rename from vllm_omni/worker/npu/npu_ar_model_runner.py rename to vllm_omni/platforms/npu/worker/npu_ar_model_runner.py index 1bb53b662..0ba57e038 100644 --- a/vllm_omni/worker/npu/npu_ar_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_model_runner.py @@ -37,7 +37,7 @@ from vllm_ascend.utils import ProfileExecuteDuration from vllm_omni.core.sched.omni_ar_scheduler import KVCacheTransferData from vllm_omni.outputs import OmniModelRunnerOutput -from vllm_omni.worker.npu.npu_model_runner import OmniNPUModelRunner +from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner class ExecuteModelState(NamedTuple): diff --git a/vllm_omni/worker/npu/npu_ar_worker.py b/vllm_omni/platforms/npu/worker/npu_ar_worker.py similarity index 74% rename from vllm_omni/worker/npu/npu_ar_worker.py rename to vllm_omni/platforms/npu/worker/npu_ar_worker.py index c4895ee1b..1fa3e59ed 100644 --- a/vllm_omni/worker/npu/npu_ar_worker.py +++ b/vllm_omni/platforms/npu/worker/npu_ar_worker.py @@ -4,10 +4,11 @@ from vllm.v1.worker.workspace import init_workspace_manager from vllm_ascend.worker.worker import NPUWorker -from vllm_omni.worker.npu.npu_ar_model_runner import NPUARModelRunner +from vllm_omni.platforms.npu.worker.npu_ar_model_runner import NPUARModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin -class NPUARWorker(NPUWorker): +class NPUARWorker(OmniWorkerMixin, NPUWorker): """NPU AR worker for thinker/talker stages in Omni model.""" def init_device(self): diff --git a/vllm_omni/worker/npu/npu_generation_model_runner.py b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py similarity index 99% rename from vllm_omni/worker/npu/npu_generation_model_runner.py rename to vllm_omni/platforms/npu/worker/npu_generation_model_runner.py index e1beef982..e8559bb46 100644 --- a/vllm_omni/worker/npu/npu_generation_model_runner.py +++ b/vllm_omni/platforms/npu/worker/npu_generation_model_runner.py @@ -27,8 +27,8 @@ from vllm_ascend.platform import NPUPlatform from vllm_ascend.utils import ProfileExecuteDuration, enable_sp, lmhead_tp_enable from vllm_omni.outputs import OmniModelRunnerOutput -from vllm_omni.worker.npu.npu_ar_model_runner import ExecuteModelState -from vllm_omni.worker.npu.npu_model_runner import OmniNPUModelRunner +from vllm_omni.platforms.npu.worker.npu_ar_model_runner import ExecuteModelState +from vllm_omni.platforms.npu.worker.npu_model_runner import OmniNPUModelRunner class NPUGenerationModelRunner(OmniNPUModelRunner): diff --git a/vllm_omni/worker/npu/npu_generation_worker.py b/vllm_omni/platforms/npu/worker/npu_generation_worker.py similarity index 72% rename from vllm_omni/worker/npu/npu_generation_worker.py rename to vllm_omni/platforms/npu/worker/npu_generation_worker.py index 588d21fc1..edbaa9f64 100644 --- a/vllm_omni/worker/npu/npu_generation_worker.py +++ b/vllm_omni/platforms/npu/worker/npu_generation_worker.py @@ -4,10 +4,11 @@ from vllm.v1.worker.workspace import init_workspace_manager from vllm_ascend.worker.worker import NPUWorker -from vllm_omni.worker.npu.npu_generation_model_runner import NPUGenerationModelRunner +from vllm_omni.platforms.npu.worker.npu_generation_model_runner import NPUGenerationModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin -class NPUGenerationWorker(NPUWorker): +class NPUGenerationWorker(OmniWorkerMixin, NPUWorker): """NPU generation worker for code2wav stage in Omni model.""" def init_device(self): diff --git a/vllm_omni/worker/npu/npu_model_runner.py b/vllm_omni/platforms/npu/worker/npu_model_runner.py similarity index 100% rename from vllm_omni/worker/npu/npu_model_runner.py rename to vllm_omni/platforms/npu/worker/npu_model_runner.py diff --git a/vllm_omni/platforms/rocm/__init__.py b/vllm_omni/platforms/rocm/__init__.py new file mode 100644 index 000000000..999dc5a08 --- /dev/null +++ b/vllm_omni/platforms/rocm/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.platforms.rocm.platform import RocmOmniPlatform + +__all__ = ["RocmOmniPlatform"] diff --git a/vllm_omni/platforms/rocm/platform.py b/vllm_omni/platforms/rocm/platform.py new file mode 100644 index 000000000..f2c2d3a85 --- /dev/null +++ b/vllm_omni/platforms/rocm/platform.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from vllm.logger import init_logger +from vllm.platforms.rocm import RocmPlatform + +from vllm_omni.diffusion.attention.backends.registry import DiffusionAttentionBackendEnum +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum + +logger = init_logger(__name__) + + +class RocmOmniPlatform(OmniPlatform, RocmPlatform): + """ROCm/AMD GPU implementation of OmniPlatform. + + Inherits all ROCm-specific implementations from vLLM's RocmPlatform, + and adds Omni-specific interfaces from OmniPlatform. + """ + + _omni_enum = OmniPlatformEnum.ROCM + + @classmethod + def get_omni_ar_worker_cls(cls) -> str: + return "vllm_omni.worker.gpu_ar_worker.GPUARWorker" + + @classmethod + def get_omni_generation_worker_cls(cls) -> str: + return "vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker" + + @classmethod + def get_diffusion_attn_backend_cls( + cls, + selected_backend: str | None, + head_size: int, + ) -> str: + from vllm._aiter_ops import is_aiter_found_and_supported + + # Check if aiter is available for Flash Attention support + # aiter currently only is supported on gfx942 and gfx950 + # https://github.com/vllm-project/vllm/blob/main/vllm/_aiter_ops.py + compute_capability = torch.cuda.get_device_capability() + major, minor = compute_capability + capability = major * 10 + minor + aiter_supported = is_aiter_found_and_supported() and 90 < capability < 100 + + if selected_backend is not None: + backend_upper = selected_backend.upper() + if backend_upper == "FLASH_ATTN" and not aiter_supported: + logger.warning( + "Flash Attention requires `aiter` library which is only supported " + "on gfx942 and gfx950. Falling back to TORCH_SDPA backend." + ) + logger.info("Defaulting to diffusion attention backend SDPA") + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + backend = DiffusionAttentionBackendEnum[backend_upper] + logger.info("Using diffusion attention backend '%s'", backend_upper) + return backend.get_path() + + # Choose to enable Flash Attention by default on ROCm + # whenever possible as it is the fastest backend + if aiter_supported: + logger.info("Defaulting to diffusion attention backend FLASH_ATTN") + return DiffusionAttentionBackendEnum.FLASH_ATTN.get_path() + + logger.info("Defaulting to diffusion attention backend SDPA") + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + + @classmethod + def get_default_stage_config_path(cls) -> str: + return "vllm_omni/platforms/rocm/stage_configs" + + @classmethod + def get_torch_device(cls, local_rank: int | None = None) -> torch.device: + if local_rank is None: + return torch.device("cuda") + return torch.device("cuda", local_rank) + + @classmethod + def get_device_count(cls) -> int: + return torch.cuda.device_count() + + @classmethod + def get_device_version(cls) -> str | None: + if torch.version.hip is not None: + hip_version = torch.version.hip + return hip_version.split("-")[0] + return None + + @classmethod + def synchronize(cls) -> None: + torch.cuda.synchronize() + + @classmethod + def get_free_memory(cls, device: torch.device | None = None) -> int: + free, _ = torch.cuda.mem_get_info(device) + return free diff --git a/vllm_omni/model_executor/stage_configs/rocm/qwen2_5_omni.yaml b/vllm_omni/platforms/rocm/stage_configs/qwen2_5_omni.yaml similarity index 94% rename from vllm_omni/model_executor/stage_configs/rocm/qwen2_5_omni.yaml rename to vllm_omni/platforms/rocm/stage_configs/qwen2_5_omni.yaml index c646aa76a..7887cd2bb 100644 --- a/vllm_omni/model_executor/stage_configs/rocm/qwen2_5_omni.yaml +++ b/vllm_omni/platforms/rocm/stage_configs/qwen2_5_omni.yaml @@ -10,7 +10,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true # Now we only support eager mode @@ -38,7 +38,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.8 enforce_eager: true @@ -66,7 +66,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen2_5OmniForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler gpu_memory_utilization: 0.15 enforce_eager: true diff --git a/vllm_omni/model_executor/stage_configs/rocm/qwen3_omni_moe.yaml b/vllm_omni/platforms/rocm/stage_configs/qwen3_omni_moe.yaml similarity index 93% rename from vllm_omni/model_executor/stage_configs/rocm/qwen3_omni_moe.yaml rename to vllm_omni/platforms/rocm/stage_configs/qwen3_omni_moe.yaml index 73f65ecb5..93e05ea9c 100644 --- a/vllm_omni/model_executor/stage_configs/rocm/qwen3_omni_moe.yaml +++ b/vllm_omni/platforms/rocm/stage_configs/qwen3_omni_moe.yaml @@ -12,7 +12,7 @@ stage_args: engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.6 enforce_eager: true @@ -42,7 +42,7 @@ stage_args: engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker + worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler gpu_memory_utilization: 0.3 enforce_eager: true @@ -73,7 +73,7 @@ stage_args: engine_args: model_stage: code2wav model_arch: Qwen3OmniMoeForConditionalGeneration - worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker + worker_type: generation scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler enforce_eager: true trust_remote_code: true diff --git a/vllm_omni/platforms/xpu/__init__.py b/vllm_omni/platforms/xpu/__init__.py new file mode 100644 index 000000000..8ec0986c2 --- /dev/null +++ b/vllm_omni/platforms/xpu/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.platforms.xpu.platform import XPUOmniPlatform + +__all__ = ["XPUOmniPlatform"] diff --git a/vllm_omni/platforms/xpu/platform.py b/vllm_omni/platforms/xpu/platform.py new file mode 100644 index 000000000..5c126c6d9 --- /dev/null +++ b/vllm_omni/platforms/xpu/platform.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from vllm.logger import init_logger +from vllm.platforms.xpu import XPUPlatform + +from vllm_omni.diffusion.attention.backends.registry import DiffusionAttentionBackendEnum +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum + +logger = init_logger(__name__) + + +class XPUOmniPlatform(OmniPlatform, XPUPlatform): + """XPU/Intel GPU implementation of OmniPlatform. + + Inherits all XPU-specific implementations from vLLM's XPUPlatform, + and adds Omni-specific interfaces from OmniPlatform. + """ + + _omni_enum = OmniPlatformEnum.XPU + + @classmethod + def get_omni_ar_worker_cls(cls) -> str: + return "vllm_omni.platforms.xpu.worker.xpu_ar_worker.XPUARWorker" + + @classmethod + def get_omni_generation_worker_cls(cls) -> str: + return "vllm_omni.platforms.xpu.worker.xpu_generation_worker.XPUGenerationWorker" + + @classmethod + def get_diffusion_attn_backend_cls( + cls, + selected_backend: str | None, + head_size: int, + ) -> str: + if selected_backend is not None: + backend_upper = selected_backend.upper() + backend = DiffusionAttentionBackendEnum[backend_upper] + logger.info("Using diffusion attention backend '%s'", backend_upper) + return backend.get_path() + + logger.info("Defaulting to diffusion attention backend SDPA") + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + + @classmethod + def get_default_stage_config_path(cls) -> str: + return "vllm_omni/platforms/xpu/stage_configs" + + @classmethod + def get_torch_device(cls, local_rank: int | None = None) -> torch.device: + if local_rank is None: + return torch.device("xpu") + return torch.device("xpu", local_rank) + + @classmethod + def get_device_count(cls) -> int: + return torch.xpu.device_count() + + @classmethod + def get_device_version(cls) -> str | None: + # XPU does not have a version string like CUDA + return None + + @classmethod + def synchronize(cls) -> None: + torch.xpu.synchronize() + + @classmethod + def get_free_memory(cls, device: torch.device | None = None) -> int: + if device is None: + device_id = 0 + else: + device_id = device.index if device.index is not None else 0 + props = torch.xpu.get_device_properties(device_id) + return props.total_memory diff --git a/vllm_omni/plugins/__init__.py b/vllm_omni/plugins/__init__.py new file mode 100644 index 000000000..41bf398ca --- /dev/null +++ b/vllm_omni/plugins/__init__.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import logging +from collections.abc import Callable +from typing import Any + +import vllm.envs as envs + +logger = logging.getLogger(__name__) + +# Default plugins group will be loaded in all processes (process0, engine core +# process and worker processes). +OMNI_DEFAULT_PLUGINS_GROUP = "vllm_omni.general_plugins" +# Platform plugins group will be loaded in all processes when +# `vllm_omni.platforms.current_omni_platform` is called and the value not +# initialized. +OMNI_PLATFORM_PLUGINS_GROUP = "vllm_omni.platform_plugins" + +# Make sure one process only loads plugins once. +omni_plugins_loaded = False + + +def load_omni_plugins_by_group(group: str) -> dict[str, Callable[[], Any]]: + from importlib.metadata import entry_points + + allowed_plugins = envs.VLLM_PLUGINS + + discovered_plugins = entry_points(group=group) + if len(discovered_plugins) == 0: + logger.debug("No plugins for group %s found.", group) + return {} + + # Check if the only discovered plugin is the default one. + is_default_group = group == OMNI_DEFAULT_PLUGINS_GROUP + # Use INFO for non-default groups and DEBUG for the default group. + log_level = logger.debug if is_default_group else logger.info + + log_level("Available plugins for group %s:", group) + for plugin in discovered_plugins: + log_level("- %s -> %s", plugin.name, plugin.value) + + if allowed_plugins is None: + log_level("All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.") + + plugins: dict[str, Callable[[], Any]] = {} + for plugin in discovered_plugins: + if allowed_plugins is None or plugin.name in allowed_plugins: + if allowed_plugins is not None: + log_level("Loading plugin %s", plugin.name) + + try: + func = plugin.load() + plugins[plugin.name] = func + except Exception: + logger.exception("Failed to load plugin %s", plugin.name) + + return plugins + + +def load_omni_general_plugins() -> None: + """WARNING: plugins can be loaded for multiple times in different + processes. They should be designed in a way that they can be loaded + multiple times without causing issues. + """ + global omni_plugins_loaded + if omni_plugins_loaded: + return + omni_plugins_loaded = True + + plugins = load_omni_plugins_by_group(group=OMNI_DEFAULT_PLUGINS_GROUP) + # General plugins: we only need to execute the loaded functions. + for func in plugins.values(): + func() + + +__all__ = [ + "load_omni_general_plugins", + "OMNI_DEFAULT_PLUGINS_GROUP", + "OMNI_PLATFORM_PLUGINS_GROUP", +] diff --git a/vllm_omni/utils/__init__.py b/vllm_omni/utils/__init__.py index 34b2545db..e69de29bb 100644 --- a/vllm_omni/utils/__init__.py +++ b/vllm_omni/utils/__init__.py @@ -1,13 +0,0 @@ -from vllm_omni.utils.platform_utils import ( - detect_device_type, - get_device_control_env_var, - is_npu, - is_rocm, -) - -__all__ = [ - "detect_device_type", - "get_device_control_env_var", - "is_npu", - "is_rocm", -] diff --git a/vllm_omni/utils/platform_utils.py b/vllm_omni/utils/platform_utils.py deleted file mode 100644 index fb47018e7..000000000 --- a/vllm_omni/utils/platform_utils.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import torch -from vllm.platforms import current_platform - - -def detect_device_type() -> str: - device_type = getattr(current_platform, "device_type", None) - if isinstance(device_type, str) and device_type: - return device_type.lower() - if torch.cuda.is_available(): - return "cuda" - if hasattr(torch, "npu") and torch.npu.is_available(): # type: ignore[attr-defined] - return "npu" - return "cpu" - - -def is_npu() -> bool: - return detect_device_type() == "npu" - - -def is_rocm() -> bool: - return current_platform.is_rocm() - - -def get_device_control_env_var() -> str: - """Return the environment variable name for device visibility control.""" - if hasattr(current_platform, "device_control_env_var"): - env_var = getattr(current_platform, "device_control_env_var", None) - if isinstance(env_var, str) and env_var: - return env_var - - device_type = detect_device_type() - if device_type == "npu": - return "ASCEND_RT_VISIBLE_DEVICES" - return "CUDA_VISIBLE_DEVICES" # fallback - - -def get_diffusion_worker_class() -> type: - """Get the appropriate diffusion WorkerProc class based on current device type. - - Returns: - The WorkerProc class for the detected device type (either NPUWorkerProc or WorkerProc). - - Raises: - ImportError: If the worker module for the detected device is not available. - """ - device_type = detect_device_type() - - if device_type == "npu": - from vllm_omni.diffusion.worker.npu.npu_worker import NPUWorkerProc - - return NPUWorkerProc - else: - # Default to GPU worker for cuda and other devices - from vllm_omni.diffusion.worker.gpu_diffusion_worker import WorkerProc - - return WorkerProc diff --git a/vllm_omni/worker/gpu_ar_worker.py b/vllm_omni/worker/gpu_ar_worker.py index d2dafce68..a1d6b4665 100644 --- a/vllm_omni/worker/gpu_ar_worker.py +++ b/vllm_omni/worker/gpu_ar_worker.py @@ -13,11 +13,12 @@ from vllm.v1.worker.utils import request_memory from vllm.v1.worker.workspace import init_workspace_manager from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin logger = init_logger(__name__) -class GPUARWorker(GPUWorker): +class GPUARWorker(OmniWorkerMixin, GPUWorker): """GPU worker for autoregressive omni model stages. Extends the base GPUWorker to initialize and manage autoregressive diff --git a/vllm_omni/worker/gpu_generation_worker.py b/vllm_omni/worker/gpu_generation_worker.py index 19f8ab84b..0aa13ed76 100644 --- a/vllm_omni/worker/gpu_generation_worker.py +++ b/vllm_omni/worker/gpu_generation_worker.py @@ -13,11 +13,12 @@ from vllm.v1.worker.utils import request_memory from vllm.v1.worker.workspace import init_workspace_manager from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin logger = init_logger(__name__) -class GPUGenerationWorker(GPUWorker): +class GPUGenerationWorker(OmniWorkerMixin, GPUWorker): """GPU Worker for Generation model (non-autoregressive waveform generation). Usage in stage config: diff --git a/vllm_omni/worker/mixins.py b/vllm_omni/worker/mixins.py new file mode 100644 index 000000000..5b25b8362 --- /dev/null +++ b/vllm_omni/worker/mixins.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Any + + +class OmniWorkerMixin: + """Mixin to ensure Omni plugins are loaded in worker processes.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + from vllm_omni.plugins import load_omni_general_plugins + + load_omni_general_plugins() From 741f7e282605a1e32a63a8631d4289605a7730df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E5=BF=97=E9=B9=8F?= Date: Wed, 28 Jan 2026 23:12:45 +0800 Subject: [PATCH 46/83] [Core]: KV Cache Transfer Encapsulation (#979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: princepride Signed-off-by: 汪志鹏 --- .../omni_connectors/test_kv_flow.py | 423 +++++++++------- .../worker/diffusion_model_runner.py | 133 +---- .../omni_connectors/kv_transfer_manager.py | 459 ++++++++++++++++++ .../stage_configs/bagel_multiconnector.yaml | 6 - vllm_omni/worker/gpu_ar_model_runner.py | 197 +------- 5 files changed, 714 insertions(+), 504 deletions(-) create mode 100644 vllm_omni/distributed/omni_connectors/kv_transfer_manager.py diff --git a/tests/distributed/omni_connectors/test_kv_flow.py b/tests/distributed/omni_connectors/test_kv_flow.py index 2bb06d4e0..8c7ff79ca 100644 --- a/tests/distributed/omni_connectors/test_kv_flow.py +++ b/tests/distributed/omni_connectors/test_kv_flow.py @@ -1,194 +1,251 @@ -import unittest -from types import SimpleNamespace -from unittest.mock import MagicMock, patch - +import pytest import torch -from vllm_omni.diffusion.models.bagel.pipeline_bagel import BagelPipeline +from tests.utils import hardware_test from vllm_omni.diffusion.request import OmniDiffusionRequest +from vllm_omni.distributed.omni_connectors.kv_transfer_manager import ( + OmniKVCacheConfig, + OmniKVTransferManager, +) from vllm_omni.inputs.data import OmniDiffusionSamplingParams -from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner -class MockGPUARModelRunner(GPUARModelRunner): - """Subclass to bypass heavy initialization.""" - - def __init__(self, kv_caches, input_batch): - self.kv_caches = kv_caches - self.input_batch = input_batch - self.device = "cpu" - self.cache_config = MagicMock() - self.cache_config.block_size = 16 - self.cache_config.cache_dtype = "auto" - self.logger = MagicMock() - - -class MockBagelPipeline(BagelPipeline): - """Subclass to bypass heavy initialization.""" - +class MockConnector: def __init__(self): - self.device = "cpu" - self.od_config = MagicMock() - self.bagel = MagicMock() - self.tokenizer = MagicMock() - self.language_model = MagicMock() - self.new_token_ids = {} + self.store = {} + + def put(self, from_stage, to_stage, put_key, data): + # The manager now passes full key as put_key + key = f"{from_stage}->{to_stage}:{put_key}" + self.store[key] = data + return True, len(str(data)), None # (success, size, metadata) + + def get(self, from_stage, to_stage, get_key, metadata=None): + # The manager now passes full key as get_key + key = f"{from_stage}->{to_stage}:{get_key}" + if key in self.store: + return self.store[key], len(str(self.store[key])) + return None -class TestKVFlow(unittest.TestCase): - def setUp(self): - # Common constants - self.num_layers = 2 - self.num_heads = 4 - self.head_dim = 16 - self.block_size = 8 - self.x = 8 # PagedAttention factor - self.seq_len = 20 - self.req_id = "req_test_1" - - def test_sender_extraction_logic(self): - """Test extraction logic in GPUARModelRunner.""" - if GPUARModelRunner is object: - self.skipTest("vLLM not installed") - - # 1. Setup KV Cache (List of tuples (K, V)) - # Shape: [num_blocks, block_size, num_heads, head_dim] matching 4D expectation - num_blocks = 10 - kv_caches = [] - for _ in range(self.num_layers): - k_cache = torch.randn(num_blocks, self.block_size, self.num_heads, self.head_dim) - v_cache = torch.randn(num_blocks, self.block_size, self.num_heads, self.head_dim) - # Stack K and V to create [2, num_blocks, block_size, n_heads, head_dim] - layer_cache = torch.stack([k_cache, v_cache], dim=0) - kv_caches.append(layer_cache) - - # 2. Setup Input Batch Mock - block_ids = [1, 3, 5] - mock_input_batch = MagicMock() - mock_input_batch.req_id_to_index = {self.req_id: 0} - mock_input_batch.block_table.get_row.return_value = block_ids - - # 3. Instantiate Runner - runner = MockGPUARModelRunner(kv_caches, mock_input_batch) - - # 4. Run Extraction - # calling _extract_kv_cache directly: (req_id, block_ids, seq_len) - result = runner._extract_kv_cache(self.req_id, block_ids, self.seq_len) - - # 5. Verify Result - self.assertIsNotNone(result) - self.assertEqual(result.request_id, self.req_id) - - # Check keys "key_cache" and "value_cache" exist (length 2) - self.assertEqual(len(result.layer_blocks["key_cache"]), 2) - self.assertEqual(len(result.layer_blocks["value_cache"]), 2) - - # Check Tensor Shape: [seq_len, num_heads, head_dim] - # result.layer_blocks["key_cache"] is a list of tensors - expected_shape = (self.seq_len, self.num_heads, self.head_dim) - self.assertEqual(result.layer_blocks["key_cache"][0].shape, expected_shape) - - return result # Return for use in next test - - def test_receiver_injection_logic(self): - """Test injection logic in BagelPipeline.""" - if BagelPipeline is object: - self.skipTest("vLLM not installed") - - # 1. Get Data from Sender Test (simulate transfer) - # Re-create data manually to be independent - key_cache = [] - value_cache = [] - expected_shape = (self.seq_len, self.num_heads, self.head_dim) - for i in range(self.num_layers): - key_cache.append(torch.randn(expected_shape)) - value_cache.append(torch.randn(expected_shape)) - - layer_blocks = {"key_cache": key_cache, "value_cache": value_cache} - - transfer_data = MagicMock() # Mock KVCacheTransferData - transfer_data.layer_blocks = layer_blocks - transfer_data.metadata = {"kv_lens": [self.seq_len], "ropes": [0]} - - # 2. Setup Request with Injected Data - sp = OmniDiffusionSamplingParams( - past_key_values=SimpleNamespace(**layer_blocks), - kv_metadata=transfer_data.metadata, - ) - - req = OmniDiffusionRequest(["test"], sp) - - # 3. Setup Pipeline - pipeline = MockBagelPipeline() - - # Mock Bagel's NaiveCache - class RealNaiveCache: # Minimal impl - def __init__(self, n): - self.key_cache = {i: None for i in range(n)} - self.value_cache = {i: None for i in range(n)} - - pipeline.bagel.config.llm_config.num_hidden_layers = self.num_layers - - captured_context = {} - - def mock_prepare_prompts(curr_kvlens, curr_rope, **kwargs): - # Capture the state passed to prepare_prompts - captured_context["kv_lens"] = curr_kvlens - captured_context["ropes"] = curr_rope - return {}, [0], [0] - - pipeline.bagel.prepare_prompts = MagicMock(side_effect=mock_prepare_prompts) - with patch("vllm_omni.diffusion.models.bagel.pipeline_bagel.NaiveCache") as MockNaiveCacheCls: - # Setup the instance returned by the constructor - mock_cache_instance = RealNaiveCache(self.num_layers) - MockNaiveCacheCls.return_value = mock_cache_instance - - # Verification of Injection Logic (Simulation) - current_cache = RealNaiveCache(self.num_layers) - - # --- Logic from Source Code --- - injected_kv = req.sampling_params.past_key_values - if isinstance(current_cache, RealNaiveCache) and hasattr(injected_kv, "key_cache"): - # Assuming injected_kv is SimpleNamespace or object with list attrs - for layer_idx in range(len(injected_kv.key_cache)): - if injected_kv.key_cache[layer_idx] is not None: - k_tensor = injected_kv.key_cache[layer_idx] - v_tensor = injected_kv.value_cache[layer_idx] - - if k_tensor.device != pipeline.device: - k_tensor = k_tensor.to(pipeline.device) - if v_tensor.device != pipeline.device: - v_tensor = v_tensor.to(pipeline.device) - - current_cache.key_cache[layer_idx] = k_tensor - current_cache.value_cache[layer_idx] = v_tensor - - self.assertTrue(torch.allclose(current_cache.key_cache[0], layer_blocks["key_cache"][0])) - self.assertTrue(torch.allclose(current_cache.value_cache[1], layer_blocks["value_cache"][1])) - - def test_integration(self): - """Simulate the flow from Sender -> Connector (Dict) -> Receiver.""" - if GPUARModelRunner is object or BagelPipeline is object: - self.skipTest("vLLM not installed") - - # 1. Sender (Extraction) - runner_test_result = self.test_sender_extraction_logic() # Get KVCacheTransferData - - # 2. Connector (Serialize/Deserialize Simulation) - # KVCacheTransferData has to_dict - data_dict = runner_test_result.to_dict() - - # 3. Receiver (Request Setup) - sp = OmniDiffusionSamplingParams( - past_key_values=data_dict["layer_blocks"], - kv_metadata=data_dict["metadata"], - ) - req = OmniDiffusionRequest(["integration_test"], sp) # noqa: F841 - - # 4. Receiver (Injection Simulation) - # Use the logic verification again - pass +@pytest.fixture +def mock_connector(): + return MockConnector() -if __name__ == "__main__": - unittest.main() +@pytest.fixture +def kv_config(): + return OmniKVCacheConfig( + connector_config={"type": "mock"}, + from_stage="stage1", + to_stage="stage2", + stage_id="stage2", # Acting as receiver for some tests + need_recv_cache=True, + need_send_cache=True, + recv_timeout=1.0, # Short timeout for tests + ) + + +@pytest.fixture +def common_constants(): + return { + "num_layers": 2, + "num_heads": 4, + "head_dim": 16, + "block_size": 8, + "seq_len": 20, + "req_id": "req_test_1", + } + + +@pytest.mark.cache +@hardware_test( + res={"cuda": "L4"}, + num_cards=2, +) +def test_manager_extraction(kv_config, mock_connector, common_constants): + """Test extraction and sending logic in OmniKVTransferManager.""" + num_layers = common_constants["num_layers"] + block_size = common_constants["block_size"] + num_heads = common_constants["num_heads"] + head_dim = common_constants["head_dim"] + seq_len = common_constants["seq_len"] + req_id = common_constants["req_id"] + + num_blocks = 10 + kv_caches = [] + for _ in range(num_layers): + k_cache = torch.randn(num_blocks, block_size, num_heads, head_dim) + v_cache = torch.randn(num_blocks, block_size, num_heads, head_dim) + # Stack K and V to create [2, num_blocks, block_size, n_heads, head_dim] + layer_cache = torch.stack([k_cache, v_cache], dim=0) + kv_caches.append(layer_cache) + + block_ids = [1, 3, 5] + finished_reqs = {req_id: {"block_ids": block_ids, "seq_len": seq_len}} + + manager = OmniKVTransferManager(kv_config) + # Mock the connector factory or injection + manager._connector = mock_connector + + processed = manager.handle_finished_requests_kv_transfer(finished_reqs, kv_caches, block_size, "float32") + + assert req_id in processed + + # Check if data was put into connector + # Manager builds full key: omni_{from}_to_{to}_kv_cache_{req_id} + full_request_id = f"omni_stage1_to_stage2_kv_cache_{req_id}" + expected_key = f"stage1->stage2:{full_request_id}" + assert expected_key in mock_connector.store + + data = mock_connector.store[expected_key] + assert data["request_id"] == req_id + assert "layer_blocks" in data + assert len(data["layer_blocks"]["key_cache"]) == num_layers + + # Verify shape of extracted tensor: [seq_len, heads, dim] + # Note: Manager detaches and moves to CPU + expected_shape = (seq_len, num_heads, head_dim) + assert data["layer_blocks"]["key_cache"][0].shape == expected_shape + + +@pytest.mark.cache +@hardware_test( + res={"cuda": "L4"}, + num_cards=2, +) +def test_manager_reception(kv_config, mock_connector, common_constants): + """Test reception and injection logic in OmniKVTransferManager.""" + num_layers = common_constants["num_layers"] + block_size = common_constants["block_size"] + num_heads = common_constants["num_heads"] + head_dim = common_constants["head_dim"] + seq_len = common_constants["seq_len"] + req_id = common_constants["req_id"] + + expected_shape = (seq_len, num_heads, head_dim) + key_cache = [torch.randn(expected_shape) for _ in range(num_layers)] + value_cache = [torch.randn(expected_shape) for _ in range(num_layers)] + + layer_blocks = {"key_cache": key_cache, "value_cache": value_cache} + metadata = { + "block_size": block_size, + "num_layers": num_layers, + "dtype": "float32", + "seq_len": seq_len, + } + + data_to_receive = { + "request_id": req_id, + "layer_blocks": layer_blocks, + "metadata": metadata, + "block_ids": [], + } + + # In setUp, from_stage="stage1", stage_id="stage2". recv_stages=("stage1", "stage2") + + manager = OmniKVTransferManager(kv_config) + manager._connector = mock_connector + + # Pre-populate connector with data + # Manager builds full key: omni_{from}_to_{to}_kv_cache_{req_id} + full_request_id = f"omni_stage1_to_stage2_kv_cache_{req_id}" + store_key = f"stage1->stage2:{full_request_id}" + mock_connector.store[store_key] = data_to_receive + + req = OmniDiffusionRequest( + prompts=["test_recv"], + sampling_params=OmniDiffusionSamplingParams(), + request_ids=[req_id], + ) + # req.need_kv_receive = True # Implicitly handled by receive_kv_cache check? No, manager doesn't check it, runner does. + # But receive_kv_cache in manager checks request_id. Which we need to fix in manager next. + success = manager.receive_kv_cache(req, target_device=torch.device("cpu")) + + assert success + assert hasattr(req, "past_key_values") + assert hasattr(req, "kv_metadata") + + assert len(req.past_key_values.key_cache) == num_layers + assert torch.allclose(req.past_key_values.key_cache[0], key_cache[0]) + assert req.kv_metadata["seq_len"] == seq_len + + +@pytest.mark.cache +@hardware_test( + res={"cuda": "L4"}, + num_cards=2, +) +def test_integration_flow(common_constants): + """Simulate extraction -> connector -> reception.""" + num_layers = common_constants["num_layers"] + block_size = common_constants["block_size"] + num_heads = common_constants["num_heads"] + head_dim = common_constants["head_dim"] + req_id = common_constants["req_id"] + + sender_config = OmniKVCacheConfig( + connector_config={"type": "mock"}, from_stage="sender", to_stage="receiver", need_send_cache=True + ) + sender_manager = OmniKVTransferManager(sender_config) + connector = MockConnector() + sender_manager._connector = connector # Shared connector + + # Create Data + num_blocks = 5 + kv_caches = [] + for _ in range(num_layers): + layer = torch.randn(2, num_blocks, block_size, num_heads, head_dim) + kv_caches.append(layer) + + finished_reqs = {req_id: {"block_ids": [0, 1], "seq_len": 10}} + + # Send + sender_manager.handle_finished_requests_kv_transfer(finished_reqs, kv_caches, block_size, "float32") + + receiver_config = OmniKVCacheConfig( + connector_config={"type": "mock"}, + from_stage="sender", + stage_id="receiver", + need_recv_cache=True, + recv_timeout=1.0, + ) + receiver_manager = OmniKVTransferManager(receiver_config) + receiver_manager._connector = connector # Share the same mock connector instance + + req = OmniDiffusionRequest( + prompts=["test_integ"], + sampling_params=OmniDiffusionSamplingParams(), + request_ids=[req_id], + ) + + # Receive + success = receiver_manager.receive_kv_cache(req) + + # Verify + assert success + assert req.past_key_values is not None + assert req.kv_metadata["seq_len"] == 10 + + +@pytest.mark.cache +@hardware_test( + res={"cuda": "L4"}, + num_cards=2, +) +def test_manager_extraction_no_connector(kv_config, common_constants): + """Test extraction when connector is unavailable (should still return IDs).""" + block_size = common_constants["block_size"] + req_id = common_constants["req_id"] + + manager = OmniKVTransferManager(kv_config) + # Force connector to be None + manager._connector = None + manager.config.connector_config = None + finished_reqs = {req_id: {"block_ids": [1, 2], "seq_len": 10}} + + processed = manager.handle_finished_requests_kv_transfer( + finished_reqs, kv_caches=[], block_size=block_size, cache_dtype="float32" + ) + + assert req_id in processed diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index 7335162b0..b60d2c2f5 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -28,8 +28,7 @@ from vllm_omni.diffusion.forward_context import set_forward_context from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.offload import apply_offload_hooks from vllm_omni.diffusion.request import OmniDiffusionRequest -from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory -from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec +from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager logger = init_logger(__name__) @@ -62,10 +61,9 @@ class DiffusionModelRunner: self.device = device self.pipeline = None self.cache_backend = None - self.connector = None - # Initialize OmniConnector after vllm_config is available (via init_device_and_model) - self._init_omni_connector() + # Initialize KV cache manager for connector management + self.kv_transfer_manager = OmniKVTransferManager.from_od_config(od_config) def load_model( self, @@ -138,126 +136,6 @@ class DiffusionModelRunner: logger.info("Model runner: Initialization complete.") - def _init_omni_connector(self) -> None: - # TODO(wzliu)! get real connector from yaml file instead of hardcode - """Initialize OmniConnector for KV cache transfer.""" - try: - connector_config = None - - # 1. Try to get from omni_kv_config (injected from YAML) - # Use self.od_config because self.vllm_config is a dummy VllmConfig without model_config - if self.od_config.omni_kv_config: - connector_config = self.od_config.omni_kv_config.get("connector_config") - - if not connector_config: - logger.warning("No OmniConnector config found, skipping initialization") - return - - logger.info(f"Initializing OmniConnector with config: {connector_config}") - - c_type = connector_config.get("type") - if not c_type: - logger.error("Connector config missing 'type'") - return - - c_extra = {k: v for k, v in connector_config.items() if k != "type"} - connector_spec = ConnectorSpec(name=c_type, extra=c_extra) - - self.connector = OmniConnectorFactory.create_connector(connector_spec) - - except Exception as e: - logger.error(f"Failed to initialize OmniConnector: {e}") - import traceback - - traceback.print_exc() - - def _receive_kv_cache_for_request(self, req: OmniDiffusionRequest) -> None: - """Receive KV cache for a request via OmniConnector.""" - # TODO(wzliu)! must get control info from stage queue instead of hardcode - if not req.request_ids: - logger.warning("Request has no ID, cannot receive KV cache") - return - request_id = req.request_ids[0] - - try: - logger.info(f"Attempting to receive KV cache for request {request_id}") - - # TODO: Key used for transfer (must match sender side) - # key = f"kv_cache_{req.request_id}" - - # Get data from connector - # Determine from_stage and to_stage dynamically - omni_kv_config = self.od_config.omni_kv_config - stage_id = omni_kv_config.get("stage_id") - engine_input_source = omni_kv_config.get("engine_input_source", []) - - to_stage = stage_id - # Default to stage_id - 1 if input source is not explicit - if engine_input_source: - from_stage = engine_input_source[0] - elif isinstance(stage_id, int): - from_stage = stage_id - 1 - else: - raise ValueError("Invalid stage id") - logger.info(f"Wait for KV cache for request {request_id} from stage {from_stage} to {to_stage}...") - - # Check if we should receive KV cache based on config - need_recv_cache = omni_kv_config.get("need_recv_cache", False) - if need_recv_cache: - # Default timeout 30 seconds to prevent infinite hanging - timeout = omni_kv_config.get("recv_timeout", 30.0) - start_time = time.time() - - while True: - get_key = f"omni_{from_stage}_to_{to_stage}_kv_cache_{request_id}" - result = self.connector.get( - from_stage=from_stage, - to_stage=to_stage, - get_key=get_key, - ) - if result: - break - - if time.time() - start_time > timeout: - logger.error(f"Timeout waiting for KV cache for request {request_id} after {timeout}s") - result = None - break - - time.sleep(0.5) - else: - logger.info(f"Skip receiving KV cache for {request_id} (need_recv_cache=False)") - result = None - - if result: - data, size = result - logger.info(f"Successfully received KV cache for {request_id}") - - # Assume data structure matches KVCacheTransferData.to_dict() - if isinstance(data, dict) and "layer_blocks" in data: - # Get layer blocks and ensure they are on the correct device - layer_blocks = data["layer_blocks"] - - # Move tensors to GPU if needed (OmniSerializer should handle tensor reconstruction) - for cache_list in [layer_blocks["key_cache"], layer_blocks["value_cache"]]: - for i, tensor in enumerate(cache_list): - if isinstance(tensor, torch.Tensor) and tensor.device != self.pipeline.device: - cache_list[i] = tensor.to(self.pipeline.device).contiguous() - from types import SimpleNamespace - - req.sampling_params.past_key_values = SimpleNamespace(**layer_blocks) - - if "metadata" in data: - req.sampling_params.kv_metadata = data["metadata"] - - else: - logger.warning(f"No KV cache received for {request_id} (timeout or empty)") - - except Exception as e: - logger.error(f"Error receiving KV cache for {request_id}: {e}") - import traceback - - traceback.print_exc() - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load weights into the pipeline.""" return self.pipeline.load_weights(weights) @@ -277,9 +155,8 @@ class DiffusionModelRunner: if len(req.prompts) == 0: raise ValueError("Cannot execute model with empty request list") - # [Omni] KV Cache Receiving Logic - if req.sampling_params.need_kv_receive and self.connector is not None: - self._receive_kv_cache_for_request(req) + # The manager handles the check for need_recv_cache internally + self.kv_transfer_manager.receive_kv_cache(req, target_device=getattr(self.pipeline, "device", None)) if req.sampling_params.generator is None and req.sampling_params.seed is not None: req.sampling_params.generator = torch.Generator(device=self.device).manual_seed(req.sampling_params.seed) diff --git a/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py b/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py new file mode 100644 index 000000000..82c06fdaf --- /dev/null +++ b/vllm_omni/distributed/omni_connectors/kv_transfer_manager.py @@ -0,0 +1,459 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unified OmniConnector and KV cache transfer management.""" + +import time +from collections.abc import Callable +from dataclasses import asdict, dataclass +from typing import Any + +import torch +from vllm.logger import init_logger + +from .factory import OmniConnectorFactory +from .utils.config import ConnectorSpec + +logger = init_logger(__name__) + + +@dataclass +class OmniKVCacheConfig: + """Configuration for OmniKVTransferManager.""" + + connector_config: dict[str, Any] | None = None + from_stage: str | None = None + to_stage: str | None = None + stage_id: str | int | None = None + engine_input_source: list[str | int] | None = None + need_recv_cache: bool = False + need_send_cache: bool = False + recv_timeout: float = 30.0 + + +@dataclass +class KVCacheTransferData: + """Container for KV cache transfer data.""" + + request_id: str + layer_blocks: dict[str, Any] + block_ids: list[int] + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary for serialization.""" + return asdict(self) + + +class OmniKVTransferManager: + """Unified management for OmniConnector and KV cache transfer. + + This class encapsulates all KV cache related operations: + - Connector initialization and lazy creation + - KV cache extraction from GPU blocks + - KV cache transfer with retry logic + - KV cache receiving with timeout + """ + + def __init__(self, config: OmniKVCacheConfig): + self.config = config + self._connector = None + + # Pre-calculate send stages (from_stage, to_stage) + self.send_stages = ( + (str(config.from_stage), str(config.to_stage)) if config.from_stage and config.to_stage else (None, None) + ) + + # Pre-calculate receive stages (from_stage, to_stage) + recv_from = config.from_stage + if config.engine_input_source: + recv_from = config.engine_input_source[0] + elif isinstance(config.stage_id, int): + recv_from = config.stage_id - 1 + + self.recv_stages = ( + (str(recv_from), str(config.stage_id)) + if recv_from is not None and config.stage_id is not None + else (None, None) + ) + + @classmethod + def _create(cls, cfg: dict | None) -> "OmniKVTransferManager": + """Create manager from raw config dict.""" + if not cfg or not isinstance(cfg, dict): + return cls(OmniKVCacheConfig()) + return cls( + OmniKVCacheConfig( + connector_config=cfg.get("connector_config"), + from_stage=cfg.get("omni_from_stage"), + to_stage=cfg.get("omni_to_stage"), + stage_id=cfg.get("stage_id"), + engine_input_source=cfg.get("engine_input_source", []), + need_recv_cache=cfg.get("need_recv_cache", False), + need_send_cache=cfg.get("need_send_cache", False), + recv_timeout=cfg.get("recv_timeout", 30.0), + ) + ) + + @classmethod + def from_model_config(cls, config: Any) -> "OmniKVTransferManager": + """Create from model config (for AR model runner).""" + return cls._create(getattr(config, "omni_kv_config", None)) + + @classmethod + def from_od_config(cls, config: Any) -> "OmniKVTransferManager": + """Create from OmniDiffusion config (for diffusion runner).""" + return cls._create(getattr(config, "omni_kv_config", None)) + + @classmethod + def from_vllm_config(cls, vllm_config: Any, model_config: Any) -> "OmniKVTransferManager": + """Create from vllm config with fallback to kv_transfer_config.""" + # Primary: omni_kv_config from model_config + omni_kv = getattr(model_config, "omni_kv_config", None) + if isinstance(omni_kv, dict): + return cls._create(omni_kv) + + # Fallback: check kv_transfer_config + kv_cfg = getattr(vllm_config, "kv_transfer_config", None) + if kv_cfg: + direct = getattr(kv_cfg, "omni_connector_config", None) + if isinstance(direct, dict) and direct: + return cls._create({"connector_config": direct}) + extra = getattr(kv_cfg, "kv_connector_extra_config", None) + if isinstance(extra, dict): + omni = extra.get("omni_connector_config") + if isinstance(omni, dict) and omni: + return cls._create({"connector_config": omni}) + + return cls(OmniKVCacheConfig()) + + @property + def connector(self): + """Lazy initialization of connector.""" + # If a previous initialization attempt failed, don't retry on every access. + if self._connector is False: + return None + + if self._connector is None: + cfg = self.config.connector_config + if cfg and (c_type := cfg.get("type")): + try: + logger.info(f"Initializing OmniConnector with config: {cfg}") + c_extra = {k: v for k, v in cfg.items() if k != "type"} + self._connector = OmniConnectorFactory.create_connector(ConnectorSpec(name=c_type, extra=c_extra)) + except Exception as e: + logger.error(f"Failed to initialize OmniConnector: {e}") + import traceback + + traceback.print_exc() + # Cache failure sentinel to avoid repeated initialization attempts in hot paths. + self._connector = False + + return self._connector if self._connector else None + + def get_connector(self): + """Get connector (compatibility wrapper for existing code).""" + return self.connector + + def handle_finished_requests_kv_transfer( + self, + finished_reqs: dict[str, dict[str, Any]], + kv_caches: list[torch.Tensor], + block_size: int, + cache_dtype: str, + request_id_resolver: Callable[[str], str] | None = None, + ) -> list[str]: + """Handle KV cache transfer for finished requests. + + This method extracts KV cache from GPU blocks and transfers them + to the downstream stage via the connector. + + Args: + finished_reqs: Dict mapping request_id to {block_ids, seq_len} + kv_caches: List of KV cache tensors per layer + block_size: Size of each cache block + cache_dtype: Data type of the cache + request_id_resolver: Optional function to resolve global request ID + + Returns: + List of request IDs that were processed + """ + if not finished_reqs: + return [] + + if not self.config.need_send_cache: + return list(finished_reqs.keys()) + + if not self.connector: + logger.warning("No connector available, skipping KV transfer but freeing resources") + return list(finished_reqs.keys()) + + logger.debug(f"Processing KV transfer for {len(finished_reqs)} requests") + + extracted_ids = [] + for req_id, data in finished_reqs.items(): + try: + seq_len = data.get("seq_len", 0) + block_ids = data.get("block_ids", []) + if not block_ids: + logger.warning(f"Request {req_id} has no block IDs, skipping") + continue + + # Extract KV cache from GPU blocks -> CPU tensors + kv_data = self._extract_kv_cache(req_id, block_ids, seq_len, kv_caches, block_size, cache_dtype) + if kv_data: + # Resolve global request ID if available + transfer_req_id = request_id_resolver(req_id) if request_id_resolver else req_id + + # Transfer to downstream stage via connector + self._transfer_kv_cache(kv_data, transfer_req_id) + + except Exception as e: + logger.error(f"Failed KV transfer for {req_id}: {e}") + finally: + extracted_ids.append(req_id) + + return extracted_ids + + def _extract_kv_cache( + self, + req_id: str, + block_ids: list[int], + seq_len: int, + kv_caches: list[torch.Tensor], + block_size: int, + cache_dtype: str, + ) -> KVCacheTransferData | None: + """Extract KV cache from GPU blocks for a single request. + + Args: + req_id: Request identifier + block_ids: List of block IDs to extract + seq_len: Sequence length + kv_caches: List of KV cache tensors per layer + block_size: Size of each cache block + cache_dtype: Data type of the cache + + Returns: + KVCacheTransferData if extraction successful, None otherwise + """ + num_layers = len(kv_caches) + key_cache: list[torch.Tensor | None] = [None] * num_layers + value_cache: list[torch.Tensor | None] = [None] * num_layers + + for layer_idx, kv_tensor in enumerate(kv_caches): + # Validate block IDs - shape: [2, num_blocks, block_size, n_heads, head_dim] + max_block = kv_tensor.shape[1] - 1 + valid_ids = [bid for bid in block_ids if 0 <= bid <= max_block] + if not valid_ids: + continue + + # Extract and reshape: [2, n_blocks, block_size, n_heads, head_dim] + # -> [2, seq_len, n_heads, head_dim] + selected = kv_tensor[:, valid_ids] # [2, n_valid, block_size, n_heads, head_dim] + n_kv, n_blks, blk_sz, n_heads, d_head = selected.shape + flat = selected.reshape(n_kv, n_blks * blk_sz, n_heads, d_head) + if seq_len < flat.shape[1]: + flat = flat[:, :seq_len] + + # Move to CPU + flat_cpu = flat.detach().cpu().contiguous() + key_cache[layer_idx] = flat_cpu[0] + value_cache[layer_idx] = flat_cpu[1] + + if not any(k is not None for k in key_cache): + return None + + return KVCacheTransferData( + request_id=req_id, + layer_blocks={"key_cache": key_cache, "value_cache": value_cache}, + block_ids=block_ids, + metadata={ + "block_size": block_size, + "num_layers": num_layers, + "dtype": str(cache_dtype), + "seq_len": seq_len, + }, + ) + + def _transfer_kv_cache(self, kv_data: KVCacheTransferData, transfer_req_id: str) -> None: + """Transfer KV cache data to downstream stage via OmniConnector. + + Args: + kv_data: The extracted KV cache data + transfer_req_id: The request ID to use for transfer + """ + from_stage, to_stage = self.send_stages + if not from_stage or not to_stage: + raise ValueError("Transfer stages (omni_from_stage, omni_to_stage) not configured") + + # Prepare data and transfer with retry + data_dict = kv_data.to_dict() + data_dict["request_id"] = transfer_req_id + + success, size, _ = self._transfer_with_retry(from_stage, to_stage, f"kv_cache_{transfer_req_id}", data_dict) + + if success: + logger.info(f"KV transfer OK: {transfer_req_id}, {size} bytes") + else: + logger.error(f"KV transfer FAILED: {transfer_req_id}") + + def _transfer_with_retry( + self, + from_stage: str, + to_stage: str, + request_id: str, + data: dict[str, Any], + max_retries: int = 3, + ) -> tuple[bool, int, dict[str, Any] | None]: + """Transfer data with retry and exponential backoff. + + Args: + from_stage: Source stage identifier + to_stage: Target stage identifier + request_id: Request identifier for the key + data: Data to transfer + max_retries: Maximum number of retry attempts + + Returns: + Tuple of (success, size, metadata) + """ + for attempt in range(max_retries): + try: + # Build the full key for connector + full_request_id = f"omni_{from_stage}_to_{to_stage}_{request_id}" + success, size, metadata = self.connector.put( + from_stage=from_stage, to_stage=to_stage, put_key=full_request_id, data=data + ) + if success: + return success, size, metadata + logger.warning(f"Transfer attempt {attempt + 1} failed for {request_id}") + except Exception as e: + logger.warning(f"Transfer attempt {attempt + 1} exception: {e}") + + if attempt < max_retries - 1: + time.sleep(0.1 * (2**attempt)) + + return False, 0, None + + @torch.inference_mode() + def receive_kv_cache_for_request( + self, + request_id: str, + target_device: torch.device | None = None, + ) -> tuple[dict[str, Any] | None, int]: + """Receive KV cache for a specific request. + + This implements the receiving logic from gpu_diffusion_model_runner.py. + + Args: + request_id: The request ID to receive KV cache for + target_device: Optional device to move tensors to + + Returns: + Tuple of (data dict, size) if successful, (None, 0) otherwise + """ + if not self.connector: + logger.warning("No connector available for receiving KV cache") + return None, 0 + + from_stage, to_stage = self.recv_stages + if not from_stage or not to_stage: + logger.warning("Receive stages not configured") + return None, 0 + + # Check if we should receive KV cache based on config + if not self.config.need_recv_cache: + logger.info(f"Skip receiving KV cache for {request_id} (need_recv_cache=False)") + return None, 0 + + timeout = self.config.recv_timeout + start_time = time.time() + + logger.info(f"Wait for KV cache for request {request_id} from stage {from_stage} to {to_stage}...") + + try: + while True: + # Build the full key for connector + full_request_id = f"omni_{from_stage}_to_{to_stage}_kv_cache_{request_id}" + result = self.connector.get( + from_stage=from_stage, + to_stage=to_stage, + get_key=full_request_id, + ) + if result: + data, size = result + logger.info(f"Successfully received KV cache for {request_id}, {size} bytes") + + # Move tensors to target device if specified + if target_device is not None and isinstance(data, dict) and "layer_blocks" in data: + layer_blocks = data["layer_blocks"] + for cache_list in [ + layer_blocks.get("key_cache", []), + layer_blocks.get("value_cache", []), + ]: + for i, tensor in enumerate(cache_list): + if isinstance(tensor, torch.Tensor) and tensor.device != target_device: + cache_list[i] = tensor.to(target_device).contiguous() + + return data, size + + if time.time() - start_time > timeout: + logger.error(f"Timeout waiting for KV cache for request {request_id} after {timeout}s") + return None, 0 + + time.sleep(0.5) + + except Exception as e: + logger.error(f"Error receiving KV cache for {request_id}: {e}") + import traceback + + traceback.print_exc() + return None, 0 + + def apply_kv_cache_to_request(self, req: Any, data: dict[str, Any]) -> None: + """Apply received KV cache data to a request object. + + Args: + req: The request object to apply KV cache to + data: The received KV cache data dictionary + """ + if isinstance(data, dict) and "layer_blocks" in data: + layer_blocks = data["layer_blocks"] + from types import SimpleNamespace + + kv_obj = SimpleNamespace(**layer_blocks) + req.past_key_values = kv_obj + + # [Omni] Also attach to sampling_params for BagelPipeline compatibility + # BagelPipeline checks req.sampling_params.past_key_values + if hasattr(req, "sampling_params") and req.sampling_params is not None: + req.sampling_params.past_key_values = kv_obj + + if "metadata" in data: + req.kv_metadata = data["metadata"] + + # Legacy compatibility method + def receive_kv_cache(self, req: Any, target_device: torch.device | None = None) -> bool: + """Receive KV cache and populate request object (legacy interface). + + Args: + req: Request object with request_id attribute + target_device: Optional device to move tensors to + + Returns: + True if successful, False otherwise + """ + request_id = getattr(req, "request_id", None) + if not request_id and hasattr(req, "request_ids") and req.request_ids: + # Adaptation for new OmniDiffusionRequest which has list of prompts/ids + request_id = req.request_ids[0] + + if not request_id: + logger.warning("Request has no ID, cannot receive KV cache") + return False + + data, size = self.receive_kv_cache_for_request(request_id, target_device) + if data: + self.apply_kv_cache_to_request(req, data) + return True + return False diff --git a/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml b/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml index 86c0330a6..e378fe675 100644 --- a/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml +++ b/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml @@ -61,13 +61,7 @@ stage_args: final_output_type: image is_comprehension: false default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 seed: 52 - detokenize: True - repetition_penalty: 1.0 input_connectors: from_stage_0: mooncake_connector diff --git a/vllm_omni/worker/gpu_ar_model_runner.py b/vllm_omni/worker/gpu_ar_model_runner.py index ecc839fd3..8440cb5b8 100644 --- a/vllm_omni/worker/gpu_ar_model_runner.py +++ b/vllm_omni/worker/gpu_ar_model_runner.py @@ -35,7 +35,7 @@ from vllm.v1.worker.gpu_model_runner import ( from vllm.v1.worker.ubatch_utils import maybe_create_ubatch_slices from vllm.v1.worker.utils import is_residual_scattered_for_sp -from vllm_omni.core.sched.omni_ar_scheduler import KVCacheTransferData +from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager from vllm_omni.outputs import OmniModelRunnerOutput from vllm_omni.worker.gpu_model_runner import OmniGPUModelRunner @@ -70,7 +70,8 @@ class GPUARModelRunner(OmniGPUModelRunner): # each model stage has their own hidden size self.hidden_size = self.model_config.hf_text_config.hidden_size self.inputs_embeds = self._make_buffer(self.max_num_tokens, self.hidden_size, dtype=self.dtype, numpy=False) - self.omni_connector = None + # Initialize KV cache manager (preserve vllm_config fallback behavior) + self.kv_transfer_manager = OmniKVTransferManager.from_vllm_config(self.vllm_config, self.model_config) def _make_buffer(self, *size, dtype, numpy=True): # Prevent ray from pinning the buffer due to large size @@ -95,7 +96,13 @@ class GPUARModelRunner(OmniGPUModelRunner): raise RuntimeError("State error: sample_tokens() must be called after execute_model() returns None.") # [Omni] Handle KV transfer BEFORE updating states (which removes finished requests) - self.kv_extracted_req_ids = self._handle_finished_requests_kv_transfer(scheduler_output) + self.kv_extracted_req_ids = self.kv_transfer_manager.handle_finished_requests_kv_transfer( + finished_reqs=getattr(scheduler_output, "finished_requests_needing_kv_transfer", {}), + kv_caches=self.kv_caches, + block_size=self.cache_config.block_size, + cache_dtype=str(self.cache_config.cache_dtype), + request_id_resolver=self._resolve_global_request_id, + ) if self.vllm_config.model_config.enable_return_routed_experts: capturer = RoutedExpertsCapturer.get_instance() @@ -577,161 +584,6 @@ class GPUARModelRunner(OmniGPUModelRunner): return async_output - def _handle_finished_requests_kv_transfer(self, scheduler_output: SchedulerOutput) -> list[str]: - """Handle KV cache transfer for finished requests. - - Returns list of request IDs that were processed (for Scheduler to free blocks). - """ - finished_reqs = getattr(scheduler_output, "finished_requests_needing_kv_transfer", {}) - if not finished_reqs: - return [] - - logger.debug(f"Processing KV transfer for {len(finished_reqs)} requests") - - extracted_ids = [] - for req_id, data in finished_reqs.items(): - try: - seq_len = data.get("seq_len", 0) - block_ids = data.get("block_ids", []) - if not block_ids: - logger.warning(f"Request {req_id} has no block IDs, skipping") - continue - - # Extract KV cache from GPU blocks -> CPU tensors - kv_data = self._extract_kv_cache(req_id, block_ids, seq_len) - if kv_data: - # Transfer to downstream stage via connector - self._transfer_kv_cache(kv_data) - - except Exception as e: - logger.error(f"Failed KV transfer for {req_id}: {e}") - finally: - extracted_ids.append(req_id) - - return extracted_ids - - def _extract_kv_cache(self, req_id: str, block_ids: list[int], seq_len: int) -> KVCacheTransferData | None: - """Extract KV cache from GPU blocks for a single request.""" - num_layers = len(self.kv_caches) - key_cache = [None] * num_layers - value_cache = [None] * num_layers - - for layer_idx, kv_tensor in enumerate(self.kv_caches): - # Validate block IDs - max_block = kv_tensor.shape[1] - 1 - valid_ids = [bid for bid in block_ids if 0 <= bid <= max_block] - if not valid_ids: - continue - - # Extract and reshape: [2, n_blocks, block_size, n_heads, head_dim] - # -> [2, seq_len, n_heads, head_dim] - selected = kv_tensor[:, valid_ids] # [2, n_valid, block_size, n_heads, head_dim] - n_kv, n_blks, blk_sz, n_heads, d_head = selected.shape - flat = selected.reshape(n_kv, n_blks * blk_sz, n_heads, d_head) - if seq_len < flat.shape[1]: - flat = flat[:, :seq_len] - - # Move to CPU - flat_cpu = flat.detach().cpu().contiguous() - key_cache[layer_idx] = flat_cpu[0] - value_cache[layer_idx] = flat_cpu[1] - - if not any(k is not None for k in key_cache): - return None - - return KVCacheTransferData( - request_id=req_id, - layer_blocks={"key_cache": key_cache, "value_cache": value_cache}, - block_ids=block_ids, - metadata={ - "block_size": self.cache_config.block_size, - "num_layers": num_layers, - "dtype": str(self.cache_config.cache_dtype), - "seq_len": seq_len, - }, - ) - - def _transfer_kv_cache(self, kv_data: KVCacheTransferData) -> None: - """Transfer KV cache data to downstream stage via OmniConnector.""" - connector = self._get_or_create_connector() - if not connector: - return - - # Resolve global request ID if available - transfer_req_id = self._resolve_global_request_id(kv_data.request_id) - from_stage, to_stage = self._detect_transfer_stages() - - # Prepare data and transfer with retry - data_dict = kv_data.to_dict() - data_dict["request_id"] = transfer_req_id - - success, size, _ = self._transfer_with_retry( - connector, from_stage, to_stage, f"kv_cache_{transfer_req_id}", data_dict - ) - - if success: - logger.info(f"KV transfer OK: {transfer_req_id}, {size} bytes") - else: - logger.error(f"KV transfer FAILED: {transfer_req_id}") - - def _get_or_create_connector(self) -> Any | None: - """Get existing connector or create one from config.""" - if self.omni_connector: - return self.omni_connector - - from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory - from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec - - config = self._get_omni_connector_config() - if not config or not isinstance(config, dict): - logger.warning("No valid OmniConnector config found") - return None - - c_type = config.get("type") - if not c_type: - logger.error("OmniConnector config missing 'type' field") - return None - - c_extra = {k: v for k, v in config.items() if k != "type"} - self.omni_connector = OmniConnectorFactory.create_connector(ConnectorSpec(name=c_type, extra=c_extra)) - return self.omni_connector - - def _get_omni_connector_config(self) -> dict[str, Any] | None: - """Get OmniConnector configuration from model config.""" - # Primary: omni_kv_config from YAML - omni_kv = getattr(self.model_config, "omni_kv_config", None) - if isinstance(omni_kv, dict): - cfg = omni_kv.get("connector_config") - if isinstance(cfg, dict) and cfg: - return cfg - - # Fallback: kv_transfer_config - kv_cfg = getattr(self.vllm_config, "kv_transfer_config", None) - if kv_cfg: - direct = getattr(kv_cfg, "omni_connector_config", None) - if isinstance(direct, dict) and direct: - return direct - extra = getattr(kv_cfg, "kv_connector_extra_config", None) - if isinstance(extra, dict): - omni = extra.get("omni_connector_config") - if isinstance(omni, dict) and omni: - return omni - - return None - - def _detect_transfer_stages(self) -> tuple[str, str]: - """Detect source and target stages for KV transfer.""" - omni_kv = getattr(self.model_config, "omni_kv_config", None) - if isinstance(omni_kv, dict): - from_s = omni_kv.get("omni_from_stage") - to_s = omni_kv.get("omni_to_stage") - if from_s and to_s: - return str(from_s), str(to_s) - - raise ValueError( - "KV transfer stages not configured. Please set 'omni_from_stage' and 'omni_to_stage' in omni_kv_config." - ) - def _resolve_global_request_id(self, req_id: str) -> str: """Resolve global request ID from request state.""" req_state = self.requests.get(req_id) @@ -747,32 +599,3 @@ class GPUARModelRunner(OmniGPUModelRunner): return global_id.decode("utf-8") return str(global_id) return req_id - - def _transfer_with_retry( - self, - connector: Any, - from_stage: str, - to_stage: str, - request_id: str, - data: dict[str, Any], - max_retries: int = 3, - ) -> tuple[bool, int, dict[str, Any] | None]: - """Transfer data with retry and exponential backoff.""" - import time - - for attempt in range(max_retries): - try: - put_key = f"omni_{from_stage}_to_{to_stage}_{request_id}" - success, size, metadata = connector.put( - from_stage=from_stage, to_stage=to_stage, put_key=put_key, data=data - ) - if success: - return success, size, metadata - logger.warning(f"Transfer attempt {attempt + 1} failed for {request_id}") - except Exception as e: - logger.warning(f"Transfer attempt {attempt + 1} exception: {e}") - - if attempt < max_retries - 1: - time.sleep(0.1 * (2**attempt)) - - return False, 0, None From 20e7310eb184e57f912c7ba4a997f7a7a8d55839 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Thu, 29 Jan 2026 11:21:24 +0800 Subject: [PATCH 47/83] [Test]Delete skip mark for amd ci test and fix CI failure (#927) Signed-off-by: wangyu31577 Signed-off-by: Hongsheng Liu Co-authored-by: wangyu31577 Co-authored-by: Hongsheng Liu --- tests/conftest.py | 39 +++------- tests/e2e/offline_inference/conftest.py | 7 +- .../stage_configs/rocm/qwen3_omni_ci.yaml | 8 +- tests/e2e/online_serving/test_qwen3_omni.py | 8 +- tests/e2e/stage_configs/qwen3_omni_ci.yaml | 2 + tests/utils.py | 76 ++++++++++++++----- 6 files changed, 78 insertions(+), 62 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ace891db7..e9b15aeee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" if "VLLM_TARGET_DEVICE" not in os.environ: os.environ["VLLM_TARGET_DEVICE"] = "cpu" +import gc import socket import subprocess import sys @@ -59,13 +60,12 @@ def clean_gpu_memory_between_tests(): _run_post_test_cleanup() -def _run_pre_test_cleanup(): - if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1": +def _run_pre_test_cleanup(enable_force=False): + if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force: print("GPU cleanup disabled") return print("Pre-test GPU status:") - _print_simple_gpu_status() num_gpus = torch.cuda.device_count() if num_gpus > 0: @@ -74,44 +74,25 @@ def _run_pre_test_cleanup(): wait_for_gpu_memory_to_clear( devices=list(range(num_gpus)), - threshold_ratio=0.1, + threshold_ratio=0.05, ) except Exception as e: print(f"Pre-test cleanup note: {e}") -def _run_post_test_cleanup(): - if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1": +def _run_post_test_cleanup(enable_force=False): + if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1" and not enable_force: + print("GPU cleanup disabled") return - import gc - if torch.cuda.is_available(): gc.collect() torch.cuda.empty_cache() print("Post-test GPU status:") - _print_simple_gpu_status() _print_gpu_processes() -def _print_simple_gpu_status(): - """Print simple GPU memory status""" - if not torch.cuda.is_available(): - print(" CUDA not available") - return - - num_devices = torch.cuda.device_count() - for device_id in range(num_devices): - try: - torch.cuda.set_device(device_id) - allocated = torch.cuda.memory_allocated(device_id) / (1024**2) - reserved = torch.cuda.memory_reserved(device_id) / (1024**2) - print(f" GPU {device_id}: Allocated: {allocated:.1f}MB, Reserved: {reserved:.1f}MB") - except Exception: - print(f" GPU {device_id}: Error reading status") - - def _print_gpu_processes(): """Print GPU information including nvidia-smi and system processes""" @@ -871,6 +852,9 @@ class OmniServer: *, env_dict: dict[str, str] | None = None, ) -> None: + _run_pre_test_cleanup(enable_force=True) + _run_post_test_cleanup(enable_force=True) + cleanup_dist_env_and_memory() self.model = model self.serve_args = serve_args self.env_dict = env_dict @@ -986,5 +970,6 @@ class OmniServer: def __exit__(self, exc_type, exc_val, exc_tb): if self.proc: self._kill_process_tree(self.proc.pid) - _run_post_test_cleanup() + _run_pre_test_cleanup(enable_force=True) + _run_post_test_cleanup(enable_force=True) cleanup_dist_env_and_memory() diff --git a/tests/e2e/offline_inference/conftest.py b/tests/e2e/offline_inference/conftest.py index 9ee0e4b74..891709837 100644 --- a/tests/e2e/offline_inference/conftest.py +++ b/tests/e2e/offline_inference/conftest.py @@ -10,7 +10,7 @@ import pytest from vllm import TextPrompt from vllm.distributed.parallel_state import cleanup_dist_env_and_memory -from tests.conftest import _run_post_test_cleanup +from tests.conftest import _run_post_test_cleanup, _run_pre_test_cleanup from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniSamplingParams from vllm_omni.outputs import OmniRequestOutput @@ -51,6 +51,9 @@ class OmniRunner: stage_configs_path: Optional path to YAML stage config file **kwargs: Additional arguments passed to Omni """ + cleanup_dist_env_and_memory() + _run_pre_test_cleanup(enable_force=True) + _run_post_test_cleanup(enable_force=True) self.model_name = model_name self.seed = seed @@ -337,7 +340,7 @@ class OmniRunner: self.close() del self.omni cleanup_dist_env_and_memory() - _run_post_test_cleanup() + _run_post_test_cleanup(enable_force=True) def close(self): """Close and cleanup the Omni instance.""" diff --git a/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml b/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml index 0488a06c9..3297347c4 100644 --- a/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml +++ b/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml @@ -2,7 +2,6 @@ # Stage 0: Thinker (multimodal understanding + text generation) # Stage 1: Talker (text embeddings → 16-layer RVQ codec codes) # Stage 2: Code2Wav (8-layer RVQ codes → audio waveform) - # The following config has been verified on 2x H100-80G GPUs. stage_args: - stage_id: 0 @@ -22,7 +21,6 @@ stage_args: enable_prefix_caching: false hf_config_name: thinker_config tensor_parallel_size: 2 - load_format: dummy final_output: true final_output_type: text is_comprehension: true @@ -52,7 +50,6 @@ stage_args: enable_prefix_caching: false distributed_executor_backend: "mp" hf_config_name: talker_config - load_format: dummy engine_input_source: [0] custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker # final_output: true @@ -60,7 +57,7 @@ stage_args: default_sampling_params: temperature: 0.9 top_k: 50 - max_tokens: 100 + max_tokens: 1000 seed: 42 detokenize: False repetition_penalty: 1.05 @@ -83,7 +80,6 @@ stage_args: distributed_executor_backend: "mp" max_num_batched_tokens: 1000000 hf_config_name: thinker_config - load_format: dummy async_scheduling: false engine_input_source: [1] custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav @@ -93,7 +89,7 @@ stage_args: temperature: 0.0 top_p: 1.0 top_k: -1 - max_tokens: 200 + max_tokens: 2000 seed: 42 detokenize: True repetition_penalty: 1.1 diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index eb134ab5e..54c36633e 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -133,7 +133,7 @@ def dummy_messages_from_video_data( def get_prompt(prompt_type="text_only"): prompts = { - "text_only": "What is the capital of China?", + "text_only": "What is the capital of China? Answer in 20 words.", "mix": "What is recited in the audio? What is in this image? Describe the video briefly.", } return prompts.get(prompt_type, prompts["text_only"]) @@ -144,9 +144,6 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.skipif( - current_omni_platform.is_rocm(), reason="Test skipped on AMD environment due to known output issues" -) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None: """ @@ -226,9 +223,6 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N assert similarity > 0.9, "The audio content is not same as the text" -@pytest.mark.skipif( - current_omni_platform.is_rocm(), reason="Test skipped on AMD environment due to known output issues" -) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_audio_001(client: openai.OpenAI, omni_server) -> None: """ diff --git a/tests/e2e/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/stage_configs/qwen3_omni_ci.yaml index d7b07ddc9..148a949d8 100644 --- a/tests/e2e/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/stage_configs/qwen3_omni_ci.yaml @@ -20,6 +20,7 @@ stage_args: engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" max_num_batched_tokens: 32768 + max_model_len: 32768 enable_prefix_caching: false hf_config_name: thinker_config tensor_parallel_size: 2 @@ -51,6 +52,7 @@ stage_args: engine_output_type: latent # Output codec codes for code2wav enable_prefix_caching: false max_num_batched_tokens: 32768 + max_model_len: 32768 distributed_executor_backend: "mp" hf_config_name: talker_config engine_input_source: [0] diff --git a/tests/utils.py b/tests/utils.py index ead286760..f5c513a4d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -77,11 +77,36 @@ def wait_for_gpu_memory_to_clear( threshold_ratio: float | None = None, timeout_s: float = 120, ) -> None: + import gc + assert threshold_bytes is not None or threshold_ratio is not None # Use nvml instead of pytorch to reduce measurement error from torch cuda # context. devices = get_physical_device_indices(devices) start_time = time.time() + + # Print waiting start information + device_list = ", ".join(str(d) for d in devices) + if threshold_bytes is not None: + threshold_str = f"{threshold_bytes / 2**30:.2f} GiB" + condition_str = f"Memory usage ≤ {threshold_str}" + else: + threshold_percent = threshold_ratio * 100 + threshold_str = f"{threshold_percent:.1f}%" + condition_str = f"Memory usage ratio ≤ {threshold_str}" + + print(f"[GPU Memory Monitor] Waiting for GPU {device_list} to free memory, Condition: {condition_str}") + + # Define the is_free function based on threshold type + if threshold_bytes is not None: + + def is_free(used, total): + return used <= threshold_bytes / 2**30 + else: + + def is_free(used, total): + return used / total <= threshold_ratio + while True: output: dict[int, str] = {} output_raw: dict[int, tuple[float, float]] = {} @@ -97,33 +122,44 @@ def wait_for_gpu_memory_to_clear( gb_used = mem_info.used / 2**30 gb_total = mem_info.total / 2**30 output_raw[device] = (gb_used, gb_total) - output[device] = f"{gb_used:.02f}/{gb_total:.02f}" + # Format to more readable form + usage_percent = (gb_used / gb_total) * 100 if gb_total > 0 else 0 + output[device] = f"{gb_used:.1f}GiB/{gb_total:.1f}GiB ({usage_percent:.1f}%)" - print("gpu memory used/total (GiB): ", end="") - for k, v in output.items(): - print(f"{k}={v}; ", end="") - print("") - - if threshold_bytes is not None: - - def is_free(used, total): - return used <= threshold_bytes / 2**30 # noqa E731 - - threshold = f"{threshold_bytes / 2**30} GiB" - else: - - def is_free(used, total): - return used / total <= threshold_ratio # noqa E731 - - threshold = f"{threshold_ratio:.2f}" + # Optimized GPU memory status print + print("[GPU Memory Status] Current usage:") + for device_id, mem_info in output.items(): + print(f" GPU {device_id}: {mem_info}") + # Calculate waiting duration dur_s = time.time() - start_time + elapsed_minutes = dur_s / 60 + + # Check if all devices meet the condition if all(is_free(used, total) for used, total in output_raw.values()): - print(f"Done waiting for free GPU memory on devices {devices=} ({threshold=}) {dur_s=:.02f}") + # Optimized completion message + print(f"[GPU Memory Freed] Devices {device_list} meet memory condition") + print(f" Condition: {condition_str}") + print(f" Wait time: {dur_s:.1f} seconds ({elapsed_minutes:.1f} minutes)") + print(" Final status:") + for device_id, mem_info in output.items(): + print(f" GPU {device_id}: {mem_info}") break + # Check timeout if dur_s >= timeout_s: - raise ValueError(f"Memory of devices {devices=} not free after {dur_s=:.02f} ({threshold=})") + raise ValueError( + f"[GPU Memory Timeout] Devices {device_list} still don't meet memory condition after {dur_s:.1f} seconds\n" + f"Condition: {condition_str}\n" + f"Current status:\n" + "\n".join(f" GPU {device}: {output[device]}" for device in devices) + ) + + # Add waiting hint (optional) + if dur_s > 10 and int(dur_s) % 10 == 0: # Show hint every 10 seconds + print(f"Waiting... Already waited {dur_s:.1f} seconds ({elapsed_minutes:.1f} minutes)") + + gc.collect() + torch.cuda.empty_cache() time.sleep(5) From 144fd6e1e0efd4376fbb46d9b6e26e681001c14f Mon Sep 17 00:00:00 2001 From: kYLe Date: Wed, 28 Jan 2026 21:46:48 -0600 Subject: [PATCH 48/83] [Bugfix][Doc]Specify Qwen3-TTS model name for each task type (#1036) Signed-off-by: Kyle Huang --- examples/online_serving/qwen3_tts/README.md | 2 ++ examples/online_serving/qwen3_tts/openai_speech_client.py | 1 + 2 files changed, 3 insertions(+) diff --git a/examples/online_serving/qwen3_tts/README.md b/examples/online_serving/qwen3_tts/README.md index 2cddd0fbf..d4626e7c0 100644 --- a/examples/online_serving/qwen3_tts/README.md +++ b/examples/online_serving/qwen3_tts/README.md @@ -41,12 +41,14 @@ python openai_speech_client.py \ # VoiceDesign: Describe the voice style python openai_speech_client.py \ + --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \ --task-type VoiceDesign \ --text "哥哥，你回来啦" \ --instructions "体现撒娇稚嫩的萝莉女声，音调偏高" # Base: Voice cloning python openai_speech_client.py \ + --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \ --task-type Base \ --text "Hello, this is a cloned voice" \ --ref-audio /path/to/reference.wav \ diff --git a/examples/online_serving/qwen3_tts/openai_speech_client.py b/examples/online_serving/qwen3_tts/openai_speech_client.py index 272e84265..71ba5b70a 100644 --- a/examples/online_serving/qwen3_tts/openai_speech_client.py +++ b/examples/online_serving/qwen3_tts/openai_speech_client.py @@ -90,6 +90,7 @@ def run_tts_generation(args) -> None: if args.x_vector_only: payload["x_vector_only_mode"] = True + print(f"Model: {args.model}") print(f"Task type: {args.task_type or 'CustomVoice'}") print(f"Text: {args.text}") print(f"Voice: {args.voice}") From d4190115f7ddfe890ec0ead52e8936d1c2fa34e6 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Thu, 29 Jan 2026 13:11:33 +0800 Subject: [PATCH 49/83] [Misc] pin version of fa3-fwd (#1051) Signed-off-by: zjy0516 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 809d1f53c..3e1149099 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "cache-dit==1.2.0", "tqdm>=4.66.0", "torchsde>=0.2.6", # Required for Stable Audio scheduler - "fa3-fwd", # flash attention 3, maintained by @ZJY0516 + "fa3-fwd==0.0.1", # flash attention 3, maintained by @ZJY0516 "openai-whisper>=20250625", "imageio[ffmpeg]>=2.37.2", "onnxruntime>=1.19.0", From 2ac141026b5237b5b1c70c53867872c919cd0536 Mon Sep 17 00:00:00 2001 From: TJian Date: Thu, 29 Jan 2026 14:32:25 +0800 Subject: [PATCH 50/83] [CI] [ROCm] Add more AMD CI tests (#1039) Signed-off-by: tjtanaa --- .../scripts/hardware_ci/run-amd-test.sh | 6 ++ .buildkite/test-amd.yaml | 76 +++++++++++-------- 2 files changed, 49 insertions(+), 33 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 5f7440a80..f86b4b5d9 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -116,6 +116,9 @@ if [[ $commands == *"--shard-id="* ]]; then --shm-size=16gb \ --group-add "$render_gid" \ --rm \ + -e MIOPEN_DEBUG_CONV_DIRECT=0 \ + -e MIOPEN_DEBUG_CONV_GEMM=0 \ + -e VLLM_ROCM_USE_AITER=1 \ -e HIP_VISIBLE_DEVICES="${GPU}" \ -e HF_TOKEN \ -e AWS_ACCESS_KEY_ID \ @@ -148,6 +151,9 @@ else --shm-size=16gb \ --group-add "$render_gid" \ --rm \ + -e MIOPEN_DEBUG_CONV_DIRECT=0 \ + -e MIOPEN_DEBUG_CONV_GEMM=0 \ + -e VLLM_ROCM_USE_AITER=1 \ -e HF_TOKEN \ -e AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY \ diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 08a3f7a1c..930ef7a40 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -8,14 +8,32 @@ steps: grade: Blocking commands: - export GPU_ARCHS=gfx942 - - export MIOPEN_DEBUG_CONV_DIRECT=0 - - export MIOPEN_DEBUG_CONV_GEMM=0 - - export VLLM_ROCM_USE_AITER=1 - - export VLLM_ROCM_USE_AITER_MHA=1 - - export VLLM_ROCM_USE_AITER_LINEAR=0 - - export VLLM_ROCM_USE_AITER_RMSNORM=0 - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py +- label: "Diffusion Images API LoRA E2E" + timeout_in_minutes: 20 + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export GPU_ARCHS=gfx942 + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py + +- label: "Diffusion Model CPU offloading Test" + timeout_in_minutes: 20 + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - export GPU_ARCHS=gfx942 + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py + - label: "Diffusion Cache Backend Test" timeout_in_minutes: 15 agent_pool: mi325_1 @@ -26,25 +44,30 @@ steps: - export GPU_ARCHS=gfx942 - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export MIOPEN_DEBUG_CONV_DIRECT=0 - - export MIOPEN_DEBUG_CONV_GEMM=0 - - export VLLM_ROCM_USE_AITER=1 - - export VLLM_ROCM_USE_AITER_MHA=1 - - export VLLM_ROCM_USE_AITER_LINEAR=0 - - export VLLM_ROCM_USE_AITER_RMSNORM=0 - pytest -s -v tests/e2e/offline_inference/test_cache_dit.py tests/e2e/offline_inference/test_teacache.py -- label: "Diffusion Parallelism Test" - timeout_in_minutes: 15 +- label: "Diffusion Sequence Parallelism Test" + timeout_in_minutes: 20 agent_pool: mi325_2 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking commands: - - export MIOPEN_DEBUG_CONV_DIRECT=0 - - export MIOPEN_DEBUG_CONV_GEMM=0 + - export GPU_ARCHS=gfx942 + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py +- label: "Diffusion Tensor Parallelism Test" + timeout_in_minutes: 20 + agent_pool: mi325_2 + depends_on: amd-build + commands: + - export GPU_ARCHS=gfx942 + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/offline_inference/test_zimage_tensor_parallel.py + - label: "Diffusion GPU Worker Test" timeout_in_minutes: 20 agent_pool: mi325_2 @@ -52,8 +75,6 @@ steps: mirror_hardwares: [amdproduction] grade: Blocking commands: - - export MIOPEN_DEBUG_CONV_DIRECT=0 - - export MIOPEN_DEBUG_CONV_GEMM=0 - pytest -s -v tests/diffusion/test_diffusion_worker.py - label: "Omni Model Test Qwen2-5-Omni" @@ -66,12 +87,6 @@ steps: - export GPU_ARCHS=gfx942 - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export MIOPEN_DEBUG_CONV_DIRECT=0 - - export MIOPEN_DEBUG_CONV_GEMM=0 - - export VLLM_ROCM_USE_AITER=1 - - export VLLM_ROCM_USE_AITER_MHA=1 - - export VLLM_ROCM_USE_AITER_LINEAR=0 - - export VLLM_ROCM_USE_AITER_RMSNORM=0 - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py - label: "Omni Model Test Qwen3-Omni" @@ -83,9 +98,10 @@ steps: commands: - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export MIOPEN_DEBUG_CONV_DIRECT=0 - - export MIOPEN_DEBUG_CONV_GEMM=0 - - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py + - export VLLM_TEST_CLEAN_GPU_MEMORY="1" + - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py + - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py + - pytest -s -v tests/e2e/online_serving/test_async_omni.py - label: "Diffusion Image Edit Test" timeout_in_minutes: 15 @@ -97,10 +113,4 @@ steps: - export GPU_ARCHS=gfx942 - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export MIOPEN_DEBUG_CONV_DIRECT=0 - - export MIOPEN_DEBUG_CONV_GEMM=0 - - export VLLM_ROCM_USE_AITER=1 - - export VLLM_ROCM_USE_AITER_MHA=1 - - export VLLM_ROCM_USE_AITER_LINEAR=0 - - export VLLM_ROCM_USE_AITER_RMSNORM=0 - pytest -s -v tests/e2e/online_serving/test_i2i_multi_image_input.py From 43c6f520ca24cab5e33a2bcb795f0cacc3eca472 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Thu, 29 Jan 2026 15:18:19 +0800 Subject: [PATCH 51/83] [Bugfix] fix qwen image layerd in dummy run (#1027) Signed-off-by: zjy0516 --- examples/offline_inference/image_to_image/image_edit.py | 3 +-- vllm_omni/diffusion/diffusion_engine.py | 9 +++++++-- vllm_omni/diffusion/models/interface.py | 1 + .../models/qwen_image/pipeline_qwen_image_layered.py | 2 ++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index 896c611b4..5ffab87f0 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -77,7 +77,7 @@ from pathlib import Path import torch from PIL import Image -from vllm_omni.diffusion.data import DiffusionParallelConfig, logger +from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput @@ -428,7 +428,6 @@ def main(): if not outputs: raise ValueError("No output generated from omni.generate()") - logger.info("Outputs: %s", outputs) # Extract images from OmniRequestOutput # omni.generate() returns list[OmniRequestOutput], extract images from request_output[0].images diff --git a/vllm_omni/diffusion/diffusion_engine.py b/vllm_omni/diffusion/diffusion_engine.py index b0b9b89db..a44596983 100644 --- a/vllm_omni/diffusion/diffusion_engine.py +++ b/vllm_omni/diffusion/diffusion_engine.py @@ -30,6 +30,11 @@ def supports_image_input(model_class_name: str) -> bool: return bool(getattr(model_cls, "support_image_input", False)) +def image_color_format(model_class_name: str) -> str: + model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name) + return getattr(model_cls, "color_format", "RGB") + + def supports_audio_output(model_class_name: str) -> bool: model_cls = DiffusionModelRegistry._try_load_model_cls(model_class_name) if model_cls is None: @@ -315,8 +320,8 @@ class DiffusionEngine: width = 1024 if supports_image_input(self.od_config.model_class_name): # Provide a dummy image input if the model supports it - - dummy_image = PIL.Image.new("RGB", (width, height), color=(0, 0, 0)) + color_format = image_color_format(self.od_config.model_class_name) + dummy_image = PIL.Image.new(color_format, (width, height)) else: dummy_image = None prompt: OmniTextPrompt = {"prompt": "dummy run", "multi_modal_data": {"image": dummy_image}} diff --git a/vllm_omni/diffusion/models/interface.py b/vllm_omni/diffusion/models/interface.py index 8e8ecbb77..abdfe1e50 100644 --- a/vllm_omni/diffusion/models/interface.py +++ b/vllm_omni/diffusion/models/interface.py @@ -10,6 +10,7 @@ from typing import ( @runtime_checkable class SupportImageInput(Protocol): support_image_input: ClassVar[bool] = True + color_format: ClassVar[str] = "RGB" # Default color format @runtime_checkable diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py index ae27ba708..1001e8a14 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py @@ -192,6 +192,8 @@ def retrieve_latents( class QwenImageLayeredPipeline(nn.Module, SupportImageInput): + color_format = "RGBA" + def __init__( self, *, From 8306c91ea9c8da550925133d016fd1b797ec2f2a Mon Sep 17 00:00:00 2001 From: NATURE Date: Thu, 29 Jan 2026 15:39:26 +0800 Subject: [PATCH 52/83] [BugFix] Fix noisy output without setting a seed in Qwen Image (#1043) Signed-off-by: natureofnature --- vllm_omni/entrypoints/openai/api_server.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index f80fe95fa..6c987f4d2 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -5,6 +5,7 @@ import multiprocessing.forkserver as forkserver import os # Image generation API imports +import random import time from argparse import Namespace from collections.abc import AsyncIterator @@ -984,6 +985,13 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request) gen_params.true_cfg_scale = request.true_cfg_scale if request.seed is not None: gen_params.seed = request.seed + else: + # If seed is not provided, generate a random one to ensure + # a proper generator is initialized in the backend. + # This fixes issues where using the default global generator + # might produce blurry images in some environments. + gen_params.seed = random.randint(0, 2**32 - 1) + request_id = f"img_gen_{int(time.time())}" logger.info(f"Generating {request.n} image(s) {size_str}") From cac6504adf05902e9377a78aa38c1ce7c72b5da6 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Thu, 29 Jan 2026 03:02:59 -0500 Subject: [PATCH 53/83] [bugfix] remove vllm speech route (#1060) Signed-off-by: linyueqian --- vllm_omni/entrypoints/openai/api_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 6c987f4d2..aedaceee3 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -736,6 +736,9 @@ async def create_chat_completion(request: ChatCompletionRequest, raw_request: Re return StreamingResponse(content=generator, media_type="text/event-stream") +_remove_route_from_router(router, "/v1/audio/speech", {"POST"}) + + @router.post( "/v1/audio/speech", dependencies=[Depends(validate_json_request)], From def39561aee26284aca299b6be38bcbc2e3280cc Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Thu, 29 Jan 2026 16:03:27 +0800 Subject: [PATCH 54/83] [Debug] Update GLM-Image Pipeline (#1049) Co-authored-by: root --- vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py b/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py index d222342b5..f5c167c32 100644 --- a/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py +++ b/vllm_omni/diffusion/models/glm_image/pipeline_glm_image.py @@ -491,7 +491,7 @@ class GlmImagePipeline(nn.Module): condition_grid = image_grid_thw[:-1] prior_token_image_embed = self.vision_language_encoder.get_image_features( inputs["pixel_values"], condition_grid - ) + ).pooler_output prior_token_image_embed = torch.cat(prior_token_image_embed, dim=0) flat_prior_token_image_ids = self.vision_language_encoder.get_image_tokens( prior_token_image_embed, condition_grid @@ -859,7 +859,7 @@ class GlmImagePipeline(nn.Module): preprocessed_images = ( None if isinstance(first_prompt, str) - else first_prompt.get("additional_information", {}).get("preprocessed_image") + else [first_prompt.get("additional_information", {}).get("preprocessed_image")] ) condition_images = ( None From 5b8ade8446cec56ac6919aa90b8c91ffe030b7f2 Mon Sep 17 00:00:00 2001 From: XU Mingshi <91017482+mxuax@users.noreply.github.com> Date: Thu, 29 Jan 2026 16:44:58 +0800 Subject: [PATCH 55/83] [Diffusion][Bugfix] Fix the flash_attn backends selection logic (#983) Signed-off-by: mxuax Signed-off-by: XU Mingshi <91017482+mxuax@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- .../attention/backends/flash_attn.py | 31 ++++-- .../attention/backends/ring/ring_globals.py | 55 ++++++++--- .../attention/backends/ring/ring_kernels.py | 98 ++++++------------- .../attention/backends/ring/ring_selector.py | 10 -- .../diffusion/attention/backends/utils/fa.py | 54 ++++++++-- .../diffusion/attention/parallel/ring.py | 19 ++-- vllm_omni/diffusion/envs.py | 22 ++++- vllm_omni/platforms/cuda/platform.py | 24 +++-- 8 files changed, 185 insertions(+), 128 deletions(-) diff --git a/vllm_omni/diffusion/attention/backends/flash_attn.py b/vllm_omni/diffusion/attention/backends/flash_attn.py index a9f2d925c..dc54acf04 100644 --- a/vllm_omni/diffusion/attention/backends/flash_attn.py +++ b/vllm_omni/diffusion/attention/backends/flash_attn.py @@ -10,8 +10,26 @@ from vllm_omni.diffusion.attention.backends.abstract import ( AttentionMetadata, ) +# Import flash attention functions with fallback chain from utils/fa.py +# FA3 (fa3_fwd_interface) -> FA3 (flash_attn_interface) -> FA2 (flash_attn) +from vllm_omni.diffusion.attention.backends.utils.fa import ( + HAS_FLASH_ATTN, + _pad_input, + _unpad_input, + _upad_input, + flash_attn_func, + flash_attn_varlen_func, +) + logger = init_logger(__name__) +if not HAS_FLASH_ATTN: + raise ImportError( + "FlashAttentionBackend requires Flash Attention. " + "Please install one of: fa3-fwd, flash-attention, or flash-attn. " + "Otherwise, use SDPA backend by setting DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA" + ) + class FlashAttentionBackend(AttentionBackend): accept_output_buffer: bool = True @@ -56,14 +74,6 @@ class FlashAttentionImpl(AttentionImpl): attn_metadata: AttentionMetadata = None, ) -> torch.Tensor: """CUDA/ROCm flash attention implementation.""" - from vllm_omni.diffusion.attention.backends.utils.fa import ( - _pad_input, - _unpad_input, - _upad_input, - flash_attn_func, - flash_attn_varlen_func, - ) - query_length = query.size(1) attention_mask = attn_metadata.attn_mask if attn_metadata is not None else None # Contains at least one padding token in the sequence @@ -92,13 +102,16 @@ class FlashAttentionImpl(AttentionImpl): out = _pad_input(out_unpad, indices_q, query.size(0), query_length) else: - out: torch.Tensor = flash_attn_func( + out = flash_attn_func( query, key, value, causal=self.causal, softmax_scale=self.softmax_scale, ) + # FA3 may return (out, lse) tuple, FA2 returns just out + if isinstance(out, tuple): + out = out[0] return out def forward_npu( diff --git a/vllm_omni/diffusion/attention/backends/ring/ring_globals.py b/vllm_omni/diffusion/attention/backends/ring/ring_globals.py index 7ec092785..d80d81fb4 100644 --- a/vllm_omni/diffusion/attention/backends/ring/ring_globals.py +++ b/vllm_omni/diffusion/attention/backends/ring/ring_globals.py @@ -3,30 +3,53 @@ # Copyright (c) 2024, Jiarui Fang. # Adapted from https://github.com/feifeibear/long-context-attention - -# test if flash_attn is available +# test if flash_attn (FA2) is available try: import flash_attn # noqa: F401 - from flash_attn.flash_attn_interface import _flash_attn_backward, _flash_attn_forward # noqa: F401 + from flash_attn.flash_attn_interface import _flash_attn_forward # noqa: F401 HAS_FLASH_ATTN = True -except ImportError: +except (ImportError, ModuleNotFoundError): HAS_FLASH_ATTN = False -try: - from flash_attn_interface import _flash_attn_backward as flash_attn_func_hopper_backward # noqa: F401 - from flash_attn_interface import _flash_attn_forward as flash_attn_forward_hopper # noqa: F401 - from flash_attn_interface import flash_attn_func as flash3_attn_func # noqa: F401 +# FA3 detection: try multiple sources (forward only, no backward needed for inference) +# Source 1: flash_attn_interface (from flash-attention source build) +# Source 2: fa3_fwd_interface (from fa3-fwd PyPI package, supports Ampere/Ada/Hopper) +# Note: FA3 high-level API may or may not return softmax_lse depending on version. +# For Ring Attention which requires LSE, we fall back to low-level API if needed. +HAS_FA3 = False +fa3_fwd_func = None # Low-level forward function (_flash_attn_forward) +fa3_attn_func = None # High-level attention function (flash_attn_func) - HAS_FLASH_ATTN_HOPPER = True -except ImportError: - HAS_FLASH_ATTN_HOPPER = False +# Try flash_attn_interface first (from flash-attention source build) +try: + from flash_attn_interface import _flash_attn_forward as fa3_fwd_func # noqa: F401 + from flash_attn_interface import flash_attn_func as fa3_attn_func # noqa: F401 + + HAS_FA3 = True +except (ImportError, ModuleNotFoundError): + pass + +# Fallback: try fa3_fwd_interface (PyPI package, supports Ampere/Ada/Hopper) +if not HAS_FA3: + try: + from fa3_fwd_interface import _flash_attn_forward as fa3_fwd_func # noqa: F401 + from fa3_fwd_interface import flash_attn_func as fa3_attn_func # noqa: F401 + + HAS_FA3 = True + except (ImportError, ModuleNotFoundError): + pass + +# Legacy aliases for backward compatibility +HAS_FLASH_ATTN_HOPPER = HAS_FA3 +flash_attn_forward_hopper = fa3_fwd_func +flash3_attn_func = fa3_attn_func try: from flashinfer.prefill import single_prefill_with_kv_cache # noqa: F401 HAS_FLASHINFER = True -except ImportError: +except (ImportError, ModuleNotFoundError): HAS_FLASHINFER = False try: @@ -34,26 +57,26 @@ try: from aiter import flash_attn_func as flash_attn_func_aiter # noqa: F401 HAS_AITER = True -except ImportError: +except (ImportError, ModuleNotFoundError): HAS_AITER = False try: import sageattention # noqa: F401 HAS_SAGE_ATTENTION = True -except ImportError: +except (ImportError, ModuleNotFoundError): HAS_SAGE_ATTENTION = False try: import spas_sage_attn # noqa: F401 HAS_SPARSE_SAGE_ATTENTION = True -except ImportError: +except (ImportError, ModuleNotFoundError): HAS_SPARSE_SAGE_ATTENTION = False try: import torch_npu # noqa: F401 HAS_NPU = True -except ImportError: +except (ImportError, ModuleNotFoundError): HAS_NPU = False diff --git a/vllm_omni/diffusion/attention/backends/ring/ring_kernels.py b/vllm_omni/diffusion/attention/backends/ring/ring_kernels.py index 2444dd894..ad61f9a20 100644 --- a/vllm_omni/diffusion/attention/backends/ring/ring_kernels.py +++ b/vllm_omni/diffusion/attention/backends/ring/ring_kernels.py @@ -6,7 +6,13 @@ import math import torch -from .ring_globals import HAS_AITER, HAS_FLASH_ATTN, HAS_FLASH_ATTN_HOPPER, HAS_FLASHINFER, HAS_NPU +from .ring_globals import ( + HAS_AITER, + HAS_FA3, + HAS_FLASH_ATTN, + HAS_FLASHINFER, + fa3_fwd_func, +) _scaled_dot_product_flash_attention = torch.ops.aten._scaled_dot_product_flash_attention _scaled_dot_product_efficient_attention = torch.ops.aten._scaled_dot_product_efficient_attention @@ -26,23 +32,11 @@ if HAS_FLASH_ATTN: import flash_attn from flash_attn.flash_attn_interface import _flash_attn_forward -if HAS_FLASH_ATTN_HOPPER: - from flash_attn_interface import _flash_attn_backward as flash_attn_func_hopper_backward - from flash_attn_interface import _flash_attn_forward as flash_attn_forward_hopper - from flash_attn_interface import flash_attn_func as flash3_attn_func -else: - flash_attn_forward_hopper = None - flash_attn_func_hopper_backward = None - flash3_attn_func = None - if HAS_FLASHINFER: from flashinfer.prefill import single_prefill_with_kv_cache _LOG2_E = math.log2(math.e) -if HAS_NPU: - import torch_npu - def pytorch_attn_forward( q: torch.Tensor, @@ -146,52 +140,35 @@ def flash_attn_forward( return block_out, block_lse -def flash_attn3_func_forward( - q, k, v, dropout_p, softmax_scale, causal, window_size, softcap, alibi_slopes, return_softmax -): - assert HAS_FLASH_ATTN_HOPPER - # current signature of flash_attn_forward_hopper: - # (q, k, v, softmax_scale, causal, window_size, descale_q=None, descale_k=None, descale_v=None, gqa_parallel=False) +def fa3_forward(q, k, v, dropout_p, softmax_scale, causal, window_size, softcap, alibi_slopes, return_softmax): + """FA3 forward pass for inference. - out, softmax_lse, *unused = flash_attn_forward_hopper( - q=q, - k=k, - v=v, - k_new=None, - v_new=None, - qv=None, - out=None, - cu_seqlens_q=None, - cu_seqlens_k=None, - cu_seqlens_k_new=None, - seqused_q=None, - seqused_k=None, - max_seqlen_q=None, - max_seqlen_k=None, - page_table=None, - kv_batch_idx=None, - leftpad_k=None, - rotary_cos=None, - rotary_sin=None, - seqlens_rotary=None, - q_descale=None, - k_descale=None, - v_descale=None, + FA3 supports Ampere, Ada, and Hopper GPUs. Dropout is ignored since FA3 is inference-only. + Uses low-level API (_flash_attn_forward) which always returns softmax_lse, + required for Ring Attention's correct accumulation. + """ + assert HAS_FA3, "FA3 is not available" + assert fa3_fwd_func is not None, "FA3 low-level API (fa3_fwd_func) not available" + + # Low-level API always returns (out, softmax_lse, S_dmask, rng_state) + out, softmax_lse, *_ = fa3_fwd_func( + q, + k, + v, softmax_scale=softmax_scale, - causal=False, - window_size=(-1, -1), - attention_chunk=0, - softcap=0.0, - rotary_interleaved=True, - scheduler_metadata=None, - num_splits=0, - pack_gqa=None, - sm_margin=0, + causal=causal, + window_size_left=window_size[0] if window_size else -1, + window_size_right=window_size[1] if window_size else -1, + softcap=softcap if softcap else 0.0, ) return out, softmax_lse +# Legacy alias for backward compatibility +flash_attn3_func_forward = fa3_forward + + def flash_attn_forward_aiter( q, k, @@ -264,20 +241,3 @@ def flashinfer_attn_forward( raise ValueError(f"Invalid input shape: {q.shape}") lse = lse / _LOG2_E return out, lse - - -def npu_attn_forward(q, k, v, softmax_scale=None, layout="BSND"): - assert HAS_NPU, "torch_npu is not available" - softmax_scale = q.shape[-1] ** (-0.5) - block_out, block_lse = torch_npu.npu_fused_infer_attention_score( - q, - k, - v, - num_heads=q.shape[-2], - input_layout=layout, - scale=softmax_scale, - softmax_lse_flag=True, - pre_tokens=65535, - next_tokens=65535, - ) - return block_out, block_lse.squeeze(dim=-1) diff --git a/vllm_omni/diffusion/attention/backends/ring/ring_selector.py b/vllm_omni/diffusion/attention/backends/ring/ring_selector.py index 56e4a2e6b..77189c61b 100644 --- a/vllm_omni/diffusion/attention/backends/ring/ring_selector.py +++ b/vllm_omni/diffusion/attention/backends/ring/ring_selector.py @@ -10,7 +10,6 @@ from functools import partial import torch from .ring_globals import ( - HAS_NPU, HAS_SAGE_ATTENTION, HAS_SPARSE_SAGE_ATTENTION, ) @@ -28,9 +27,6 @@ if HAS_SAGE_ATTENTION: if HAS_SPARSE_SAGE_ATTENTION: from spas_sage_attn.autotune import SparseAttentionMeansim -if HAS_NPU: - from torch_npu import npu_fused_infer_attention_score - class AttnType(Enum): AITER = "aiter" @@ -44,7 +40,6 @@ class AttnType(Enum): SAGE_FP8 = "sage_fp8" SAGE_FP8_SM90 = "sage_fp8_sm90" SPARSE_SAGE = "sparse_sage" - NPU = "npu" @classmethod def from_string(cls, s: str): @@ -157,11 +152,6 @@ def select_flash_attn_impl( return fn - elif impl_type == AttnType.NPU: - if not HAS_NPU: - raise ImportError("torch_npu is not available!") - return npu_fused_infer_attention_score - elif attn_processor is not None: return attn_processor diff --git a/vllm_omni/diffusion/attention/backends/utils/fa.py b/vllm_omni/diffusion/attention/backends/utils/fa.py index ea12bc0f1..3344c9963 100644 --- a/vllm_omni/diffusion/attention/backends/utils/fa.py +++ b/vllm_omni/diffusion/attention/backends/utils/fa.py @@ -17,17 +17,53 @@ import torch.nn.functional as F from vllm_omni.platforms import current_omni_platform -if current_omni_platform.is_rocm(): - from vllm._aiter_ops import is_aiter_found_and_supported +# Flash Attention function detection with fallback chain +flash_attn_func = None +flash_attn_varlen_func = None - # Choose to enable this by default on ROCm - # Whenever possible as it is the fastest backend - if is_aiter_found_and_supported(): - from aiter import flash_attn_func, flash_attn_varlen_func # noqa: F401 - else: - raise ImportError("Aiter is not found and supported on currentROCm device.") +if current_omni_platform.is_rocm(): + # ROCm: try Aiter first + try: + from vllm._aiter_ops import is_aiter_found_and_supported + + if is_aiter_found_and_supported(): + from aiter import flash_attn_func, flash_attn_varlen_func # noqa: F401 + except (ImportError, ModuleNotFoundError): + pass else: - from fa3_fwd_interface import flash_attn_func, flash_attn_varlen_func # noqa: F401 + # CUDA: try FA3 -> FA2 fallback chain + # Try FA3 from fa3-fwd PyPI package + try: + from fa3_fwd_interface import flash_attn_func, flash_attn_varlen_func # noqa: F401 + except (ImportError, ModuleNotFoundError): + pass + + # Fallback: Try FA3 from flash-attention source build + if flash_attn_func is None: + try: + from flash_attn_interface import flash_attn_func, flash_attn_varlen_func # noqa: F401 + except (ImportError, ModuleNotFoundError): + pass + + # Fallback: Try FA2 from flash-attn package (try multiple import paths) + if flash_attn_func is None: + try: + from flash_attn import flash_attn_func, flash_attn_varlen_func # noqa: F401 + except (ImportError, ModuleNotFoundError): + pass + + if flash_attn_func is None: + try: + from flash_attn.flash_attn_interface import ( # noqa: F401 + flash_attn_func, + flash_attn_varlen_func, + ) + except (ImportError, ModuleNotFoundError): + pass + +# If no FA backend available, SDPA backend will be selected at the platform level +# flash_attn_func and flash_attn_varlen_func will be None +HAS_FLASH_ATTN = flash_attn_func is not None def _index_first_axis(tensor, indices): diff --git a/vllm_omni/diffusion/attention/parallel/ring.py b/vllm_omni/diffusion/attention/parallel/ring.py index 2d63f40c0..090d2ef61 100644 --- a/vllm_omni/diffusion/attention/parallel/ring.py +++ b/vllm_omni/diffusion/attention/parallel/ring.py @@ -10,7 +10,7 @@ import torch from vllm.logger import init_logger # import torch.distributed as dist # Not used directly here, but good practice if needed -from vllm_omni.diffusion.attention.backends.ring.ring_globals import HAS_FLASH_ATTN +from vllm_omni.diffusion.attention.backends.ring.ring_globals import HAS_FA3, HAS_FLASH_ATTN from vllm_omni.diffusion.attention.backends.ring.ring_selector import AttnType from vllm_omni.diffusion.attention.parallel.base import ( ParallelAttentionContext, @@ -116,11 +116,14 @@ class RingParallelAttention: except Exception: backend_pref = None - # Fallback for FP32 or if Flash Attention is not available - if query.dtype == torch.float32 or not HAS_FLASH_ATTN: - if not HAS_FLASH_ATTN and backend_pref != "sdpa": + # Determine attention type with fallback chain: FA3 -> FA2 -> SDPA + # FP32 is not supported by Flash Attention, force SDPA + if query.dtype == torch.float32: + backend_pref = "sdpa" + elif not HAS_FA3 and not HAS_FLASH_ATTN: + if backend_pref != "sdpa": logger = init_logger(__name__) - logger.warning_once("Flash Attention is not available! Force enabling SDPA.") + logger.warning_once("Flash Attention (FA2/FA3) is not available! Force enabling SDPA.") backend_pref = "sdpa" # Extract joint tensors @@ -150,6 +153,9 @@ class RingParallelAttention: from vllm_omni.diffusion.attention.backends.ring_flash_attn import ring_flash_attn_func + # Prefer FA3 over FA2 for better performance (FA3 supports Ampere/Ada/Hopper) + attn_type = AttnType.FA3 if HAS_FA3 else AttnType.FA + return ring_flash_attn_func( query, key, @@ -161,9 +167,8 @@ class RingParallelAttention: softcap=0.0, alibi_slopes=None, deterministic=False, - # return_attn_probs=False, # Removed as it might not be supported in signature group=self._sp_group.ring_group, - attn_type=AttnType.FA, + attn_type=attn_type, joint_tensor_key=joint_key, joint_tensor_value=joint_value, joint_strategy=joint_strategy, diff --git a/vllm_omni/diffusion/envs.py b/vllm_omni/diffusion/envs.py index be2b58ae8..dd566b059 100644 --- a/vllm_omni/diffusion/envs.py +++ b/vllm_omni/diffusion/envs.py @@ -66,14 +66,32 @@ class PackagesEnvChecker: if "Turing" in gpu_name or "Tesla" in gpu_name or "T4" in gpu_name: return False + # Check for any FA backend: FA3 (fa3_fwd_interface, flash_attn_interface) or FA2 (flash_attn) + # Try FA3 from fa3-fwd PyPI package + try: + import fa3_fwd_interface # noqa: F401 + + return True + except (ImportError, ModuleNotFoundError): + pass + + # Try FA3 from flash-attention source build + try: + import flash_attn_interface # noqa: F401 + + return True + except (ImportError, ModuleNotFoundError): + pass + + # Try FA2 from flash-attn package from flash_attn import __version__ if __version__ < "2.6.0": raise ImportError("install flash_attn >= 2.6.0") return True - except ImportError: + except (ImportError, ModuleNotFoundError): if not packages_info.get("has_aiter", False): - logger.warning('Flash Attention library "flash_attn" not found, using pytorch attention implementation') + logger.warning("No Flash Attention backend found, using pytorch SDPA implementation") return False def get_packages_info(self) -> dict: diff --git a/vllm_omni/platforms/cuda/platform.py b/vllm_omni/platforms/cuda/platform.py index 9c65877be..89e403f99 100644 --- a/vllm_omni/platforms/cuda/platform.py +++ b/vllm_omni/platforms/cuda/platform.py @@ -39,22 +39,34 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): selected_backend: str | None, head_size: int, ) -> str: + from vllm_omni.diffusion.envs import PACKAGES_CHECKER + # Check compute capability for Flash Attention support # Flash Attention requires compute capability >= 8.0 and < 10.0 compute_capability = cls.get_device_capability() - flash_attn_supported = False + compute_supported = False if compute_capability is not None: major, minor = compute_capability capability = major * 10 + minor - flash_attn_supported = 80 <= capability < 100 + compute_supported = 80 <= capability < 100 + + # Check if FA packages are available + packages_info = PACKAGES_CHECKER.get_packages_info() + packages_available = packages_info.get("has_flash_attn", False) + + # Both compute capability and packages must be available for FA + flash_attn_supported = compute_supported and packages_available if selected_backend is not None: backend_upper = selected_backend.upper() if backend_upper == "FLASH_ATTN" and not flash_attn_supported: - logger.warning( - "Flash Attention requires GPU with compute capability >= 8.0 " - "and < 10.0. Falling back to TORCH_SDPA backend." - ) + if not compute_supported: + logger.warning( + "Flash Attention requires GPU with compute capability >= 8.0 " + "and < 10.0. Falling back to TORCH_SDPA backend." + ) + elif not packages_available: + logger.warning("Flash Attention packages not available. Falling back to TORCH_SDPA backend.") logger.info("Defaulting to diffusion attention backend SDPA") return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() backend = DiffusionAttentionBackendEnum[backend_upper] From dd518973de11e610b427d50c0ebb851ac92a61f6 Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Thu, 29 Jan 2026 19:30:11 +0800 Subject: [PATCH 56/83] [BugFix] Fix the accuracy issue of multimodal input. (#1020) Signed-off-by: amy-why-3459 Co-authored-by: Rein Yang --- tests/conftest.py | 30 +++++++ tests/e2e/online_serving/test_qwen3_omni.py | 16 +--- .../distributed/omni_connectors/adapter.py | 9 +- vllm_omni/engine/output_processor.py | 2 +- vllm_omni/entrypoints/async_omni.py | 88 +++++++++---------- vllm_omni/entrypoints/openai/serving_chat.py | 8 +- .../models/qwen3_omni/qwen3_omni_code2wav.py | 2 +- 7 files changed, 87 insertions(+), 68 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e9b15aeee..7737a7dc2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -604,6 +604,36 @@ def convert_audio_to_text(audio_data): return "" +def merge_base64_and_convert_to_text(base64_list): + """ + Merge a list of base64 encoded audio data and convert to text. + """ + import whisper + from pydub import AudioSegment + + merged_audio = None + for base64_data in base64_list: + audio_data = base64.b64decode(base64_data.split(",", 1)[-1]) + seg = AudioSegment.from_file(io.BytesIO(audio_data)) + if merged_audio is None: + merged_audio = seg + else: + merged_audio += seg + output_path = f"./test_{int(time.time())}" + merged_audio.export(output_path, format="wav") + model = whisper.load_model("base") + text = model.transcribe( + output_path, + temperature=0.0, + word_timestamps=True, + condition_on_previous_text=False, + )["text"] + if text: + return text + else: + return "" + + def modify_stage_config( yaml_path: str, updates: dict[str, Any], diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index 54c36633e..073419fb8 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -25,6 +25,7 @@ from tests.conftest import ( generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video, + merge_base64_and_convert_to_text, modify_stage_config, ) from vllm_omni.platforms import current_omni_platform @@ -154,11 +155,6 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N Input Setting: stream=True Datasets: single request """ - # TODO：This skip will be removed when the chunk scenario supports multimodal input. - param = request.node.callspec.params.get("omni_server") - - if param[1] == CHUNK_CONFIG_PATH: - pytest.skip("The current chunk scenario does not support multimodal.") # Test single completion e2e_list = list() @@ -178,7 +174,7 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True) text_content = "" - audio_data = None + audio_data = [] for chunk in chat_completion: for choice in chunk.choices: if hasattr(choice, "delta"): @@ -189,11 +185,7 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N modality = getattr(chunk, "modality", None) if modality == "audio" and content: - # Audio chunk - content - if audio_data is None: - audio_data = content - else: - audio_data += content + audio_data.append(content) elif modality == "text" and content: # Text chunk - accumulate text content text_content += content if content else "" @@ -215,7 +207,7 @@ def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> N ), "The output does not contain any of the keywords." # Verify text output same as audio output - audio_content = convert_audio_to_text(audio_data) + audio_content = merge_base64_and_convert_to_text(audio_data) print(f"text content is: {text_content}") print(f"audio content is: {audio_content}") similarity = cosine_similarity_text(audio_content.lower(), text_content.lower()) diff --git a/vllm_omni/distributed/omni_connectors/adapter.py b/vllm_omni/distributed/omni_connectors/adapter.py index a634595ac..b03202525 100644 --- a/vllm_omni/distributed/omni_connectors/adapter.py +++ b/vllm_omni/distributed/omni_connectors/adapter.py @@ -247,7 +247,7 @@ def get_through_connector(connector, target_stage_id, stage_id, req_id, connecto connector.get_requests[req_id] += 1 logger.debug(f"[Stage-{stage_id}] Received one chunk for request {connector_get_key}") break - time.sleep(0.1) + time.sleep(0.01) return payload_data @@ -330,10 +330,11 @@ def put_chunk( if stage_id == 0 and chunk_id == 0: if connector.request_payload.get(request_id) is None: - connector.request_payload[request_id] = payload_data - return + if not payload_data.get("finished"): + connector.request_payload[request_id] = payload_data + return else: - save_payload = connector.request_payload.get(request_id) + save_payload = connector.request_payload.pop(request_id) payload_data["thinker_embeddings"] = torch.cat( (save_payload.get("thinker_embeddings"), payload_data.get("thinker_embeddings")), dim=0 ) diff --git a/vllm_omni/engine/output_processor.py b/vllm_omni/engine/output_processor.py index ab5cf878c..65a69e0d4 100644 --- a/vllm_omni/engine/output_processor.py +++ b/vllm_omni/engine/output_processor.py @@ -114,7 +114,7 @@ class OmniRequestState(RequestState): if k == "audio": # When the audio tensor shape is inconsistent, torch.cat will fail. # We need to use torch.cat in -1 dimension. - self.mm_accumulated[k] = torch.cat(v, dim=-1) + continue else: self.mm_accumulated[k] = torch.cat(v, dim=0) except Exception: diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index 179f58ed2..ec2a18ac4 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -1,12 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import copy import time import weakref from collections.abc import AsyncGenerator, Iterable, Sequence from dataclasses import asdict from pprint import pformat -from typing import Any, cast +from typing import Any from vllm.config import VllmConfig from vllm.inputs.preprocess import InputPreprocessor @@ -32,7 +33,7 @@ from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load from vllm_omni.entrypoints.utils import ( get_final_stage_id_for_e2e, ) -from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams, OmniTokensPrompt +from vllm_omni.inputs.data import OmniPromptType, OmniSamplingParams # Internal imports (our code) from vllm_omni.lora.request import LoRARequest @@ -304,38 +305,25 @@ class AsyncOmni(OmniBase): req_state = ClientRequestState(request_id) req_state.metrics = metrics self.request_states[request_id] = req_state - + sp0: SamplingParams = sampling_params_list[0] # type: ignore[index] + task = { + "request_id": request_id, + "engine_inputs": prompt, + "sampling_params": sp0, + } + self.stage_list[0].submit(task) + metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time() + _req_start_ts[request_id] = time.time() + logger.info( + f"[{self._name}] Entering scheduling loop: stages={num_stages}, final_stage={final_stage_id_for_e2e}" + ) if self.async_chunk: stage_queues = {stage_id: asyncio.Queue() for stage_id in range(num_stages)} req_state.stage_queues = stage_queues - for i in range(num_stages): - sp: SamplingParams = cast(SamplingParams, sampling_params_list[i]) - engine_inputs = cast(OmniTokensPrompt, prompt) - if i != 0: - prompt_token_ids = engine_inputs["prompt_token_ids"] - prompt_1 = engine_inputs.copy() - prompt_1["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids) - prompt_1["multi_modal_data"] = prompt_1["mm_processor_kwargs"] = None - engine_inputs = prompt_1 - - task = { - "request_id": request_id, - "engine_inputs": engine_inputs, - "sampling_params": sp, - } - self.stage_list[i].submit(task) - metrics.stage_first_ts[i] = metrics.stage_first_ts[0] or time.time() - - logger.info(f"[{self._name}] Enqueued request {request_id} to stage-{str(i)}") - - _req_start_ts[request_id] = time.time() - - logger.info( - f"[{self._name}] Entering scheduling loop: " - f"stages={num_stages}, final_stage={final_stage_id_for_e2e}" - ) async for output in self._process_async_results( request_id, + prompt, + sampling_params_list, req_state, metrics, final_stage_id_for_e2e, @@ -344,22 +332,6 @@ class AsyncOmni(OmniBase): ): yield output else: - sp0: SamplingParams = sampling_params_list[0] # type: ignore[index] - task = { - "request_id": request_id, - "engine_inputs": prompt, - "sampling_params": sp0, - } - self.stage_list[0].submit(task) - - _req_start_ts[request_id] = time.time() - # Mark first input time for stage-0 - metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time() - logger.info( - f"[{self._name}] Entering scheduling loop: " - f"stages={num_stages}, final_stage={final_stage_id_for_e2e}" - ) - async for output in self._process_sequential_results( request_id, req_state, @@ -390,6 +362,8 @@ class AsyncOmni(OmniBase): async def _process_async_results( self, request_id: str, + prompt: Any, + sampling_params_list: list[SamplingParams], req_state: ClientRequestState, metrics: OrchestratorMetrics, final_stage_id_for_e2e: int, @@ -397,16 +371,34 @@ class AsyncOmni(OmniBase): wall_start_ts: float, ) -> AsyncGenerator[OmniRequestOutput, None]: all_stages_finished = {stage_id: False for stage_id in range(final_stage_id_for_e2e + 1)} + submit_flag = True while not all(all_stages_finished.values()): for stage_id, stage in enumerate(self.stage_list[: final_stage_id_for_e2e + 1]): if all_stages_finished[stage_id]: continue - result = await req_state.stage_queues[stage_id].get() - logger.info(f"[{self._name}] Received result from stage-{stage_id}: {result}") + try: + result = req_state.stage_queues[stage_id].get_nowait() + except asyncio.QueueEmpty: + await asyncio.sleep(0.001) + continue + engine_outputs, finished, output_to_yield = self._process_single_result( result, stage, stage_id, metrics, req_start_ts, wall_start_ts, final_stage_id_for_e2e ) - + if submit_flag and stage_id == 0: + submit_flag = False + prompt_token_ids = engine_outputs.prompt_token_ids + engine_input = copy.deepcopy(prompt) + engine_input["prompt_token_ids"] = [0] * compute_talker_prompt_ids_length(prompt_token_ids) + engine_input["multi_modal_data"] = engine_input["mm_processor_kwargs"] = None + for i in range(1, len(self.stage_list)): + task = { + "request_id": request_id, + "engine_inputs": engine_input, + "sampling_params": sampling_params_list[i], + } + self.stage_list[i].submit(task) + metrics.stage_first_ts[i] = time.time() all_stages_finished[stage_id] = finished if output_to_yield: diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index db8c972a1..61b00d803 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -9,6 +9,7 @@ from io import BytesIO from typing import TYPE_CHECKING, Any, Final, Optional, cast import jinja2 +import torch from fastapi import Request from PIL import Image from pydantic import TypeAdapter @@ -1656,10 +1657,13 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): ): choices: list[ChatCompletionResponseChoice] = [] final_res = omni_outputs.request_output + audio_data = final_res.multimodal_output.get("audio") if stream: - audio_tensor = final_res.multimodal_output["audio"][-1].float().detach().cpu().numpy() + audio_tensor = audio_data[-1].float().detach().cpu().numpy() else: - audio_tensor = final_res.multimodal_output["audio"].float().detach().cpu().numpy() + if isinstance(audio_data, list): + audio_data = torch.cat(audio_data, dim=-1) + audio_tensor = audio_data.float().detach().cpu().numpy() # Ensure audio is 1D (flatten if needed) if audio_tensor.ndim > 1: diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py index bc2242f70..7adb6f96f 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py @@ -219,7 +219,7 @@ class Qwen3OmniMoeCode2Wav(nn.Module): end_index = codes.shape[-1] # TODO: need to optimize algorithms, current only support # chunk_size = left_context_size = 25 - if end_index == chunk_size: + if end_index <= chunk_size: context_size = 0 else: context_size = left_context_size From 0c8b5ac2d587f97db5709346457d597702735fbc Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Thu, 29 Jan 2026 20:02:39 +0800 Subject: [PATCH 57/83] [Bugfix] Set VaeImageProcessor `do_convert_rgb` True (#1032) Signed-off-by: gcanlin --- .../diffusion/models/qwen_image/pipeline_qwen_image_edit.py | 6 +++--- .../models/qwen_image/pipeline_qwen_image_edit_plus.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py index ba65bf07e..4241370ba 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py @@ -61,7 +61,7 @@ def get_qwen_image_edit_pre_process_func( vae_config = json.load(f) vae_scale_factor = 2 ** len(vae_config["temporal_downsample"]) if "temporal_downsample" in vae_config else 8 - image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2, do_convert_rgb=True) latent_channels = vae_config.get("z_dim", 16) def pre_process_func( @@ -136,7 +136,7 @@ def get_qwen_image_edit_post_process_func( vae_config = json.load(f) vae_scale_factor = 2 ** len(vae_config["temporal_downsample"]) if "temporal_downsample" in vae_config else 8 - image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2, do_convert_rgb=True) def post_process_func( images: torch.Tensor, @@ -261,7 +261,7 @@ class QwenImageEditPipeline( self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 self.latent_channels = self.vae.config.z_dim if getattr(self, "vae", None) else 16 - self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2, do_convert_rgb=True) self.tokenizer_max_length = 1024 # Edit prompt template - different from generation template self.prompt_template_encode = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" # noqa: E501 diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py index 52f1d64ec..9402c1f7c 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py @@ -67,7 +67,7 @@ def get_qwen_image_edit_plus_pre_process_func( vae_config = json.load(f) vae_scale_factor = 2 ** len(vae_config["temporal_downsample"]) if "temporal_downsample" in vae_config else 8 - image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2, do_convert_rgb=True) latent_channels = vae_config.get("z_dim", 16) def pre_process_func( @@ -158,7 +158,7 @@ def get_qwen_image_edit_plus_post_process_func( vae_config = json.load(f) vae_scale_factor = 2 ** len(vae_config["temporal_downsample"]) if "temporal_downsample" in vae_config else 8 - image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2, do_convert_rgb=True) def post_process_func( images: torch.Tensor, @@ -214,7 +214,7 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 self.latent_channels = self.vae.config.z_dim if getattr(self, "vae", None) else 16 - self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2, do_convert_rgb=True) self.tokenizer_max_length = 1024 # Edit prompt template - different from generation template, supports multiple images self.prompt_template_encode = ( From ee0f7a7797bb0dd9ac0befb9594eb18990e38fc1 Mon Sep 17 00:00:00 2001 From: JohnJan Date: Thu, 29 Jan 2026 20:19:31 +0800 Subject: [PATCH 58/83] [feat]: adapt batch request for flux (#1028) Signed-off-by: wuzhongjian wuzhongjian_yewu@cmss.chinamobile.com --- .../diffusion/models/flux/pipeline_flux.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/vllm_omni/diffusion/models/flux/pipeline_flux.py b/vllm_omni/diffusion/models/flux/pipeline_flux.py index 01e0bfba9..a91601b7e 100644 --- a/vllm_omni/diffusion/models/flux/pipeline_flux.py +++ b/vllm_omni/diffusion/models/flux/pipeline_flux.py @@ -557,10 +557,10 @@ class FluxPipeline( def forward( self, req: OmniDiffusionRequest, - prompt: str | list[str] = None, - prompt_2: str | list[str] = None, - negative_prompt: str | list[str] = None, - negative_prompt_2: str | list[str] = None, + prompt: str | list[str] | None = None, + prompt_2: str | list[str] | None = None, + negative_prompt: str | list[str] | None = None, + negative_prompt_2: str | list[str] | None = None, true_cfg_scale: float = 1.0, height: int | None = None, width: int | None = None, @@ -581,16 +581,27 @@ class FluxPipeline( max_sequence_length: int = 512, ): """Forward pass for flux.""" - prompt = req.prompt if req.prompt is not None else prompt - negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt - height = req.height or self.default_sample_size * self.vae_scale_factor - width = req.width or self.default_sample_size * self.vae_scale_factor - sigmas = req.sigmas or sigmas - num_inference_steps = req.num_inference_steps or num_inference_steps - generator = req.generator or generator - req_num_outputs = getattr(req, "num_outputs_per_prompt", None) - if req_num_outputs and req_num_outputs > 0: - num_images_per_prompt = req_num_outputs + # TODO: In online mode, sometimes it receives [{"negative_prompt": None}, {...}], so cannot use .get("...", "") + # TODO: May be some data formatting operations on the API side. Hack for now. + prompt = [p if isinstance(p, str) else (p.get("prompt") or "") for p in req.prompts] or prompt + if all(isinstance(p, str) or p.get("negative_prompt") is None for p in req.prompts): + negative_prompt = None + elif req.prompts: + negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] + + height = req.sampling_params.height or self.default_sample_size * self.vae_scale_factor + width = req.sampling_params.width or self.default_sample_size * self.vae_scale_factor + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps + sigmas = req.sampling_params.sigmas or sigmas + guidance_scale = ( + req.sampling_params.guidance_scale if req.sampling_params.guidance_scale is not None else guidance_scale + ) + generator = req.sampling_params.generator or generator + num_images_per_prompt = ( + req.sampling_params.num_outputs_per_prompt + if req.sampling_params.num_outputs_per_prompt > 0 + else num_images_per_prompt + ) # 1. Check inputs. Raise error if not correct self.check_inputs( From c48a9f33adc950aa0af7d2c71ce83aa31f0eaf37 Mon Sep 17 00:00:00 2001 From: Ziming Huang Date: Thu, 29 Jan 2026 21:10:09 +0800 Subject: [PATCH 59/83] [CI] Change Qwen3 Omni stage placement strategy (#1072) Signed-off-by: ZeldaHuang --- .../offline_inference/stage_configs/qwen3_omni_ci.yaml | 10 +++++----- .../stage_configs/rocm/qwen3_omni_ci.yaml | 10 +++++----- .../online_serving/stage_configs/qwen3_omni_ci.yaml | 10 +++++----- .../stage_configs/rocm/qwen3_omni_ci.yaml | 10 +++++----- tests/e2e/stage_configs/qwen3_omni_ci.yaml | 10 +++++----- .../model_executor/stage_configs/qwen3_omni_moe.yaml | 10 +++++----- .../stage_configs/qwen3_omni_moe_async_chunk.yaml | 10 +++++----- .../stage_configs/qwen3_omni_moe_multiconnector.yaml | 10 +++++----- .../platforms/rocm/stage_configs/qwen3_omni_moe.yaml | 10 +++++----- 9 files changed, 45 insertions(+), 45 deletions(-) diff --git a/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml index 7ac435ea5..477e6e59f 100644 --- a/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml @@ -7,21 +7,21 @@ stage_args: - stage_id: 0 runtime: - devices: "0,1" + devices: "0" max_batch_size: 1 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" enable_prefix_caching: false hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 load_format: dummy final_output: true final_output_type: text @@ -44,7 +44,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -68,7 +68,7 @@ stage_args: - stage_id: 2 runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav diff --git a/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml b/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml index 7ac435ea5..477e6e59f 100644 --- a/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml +++ b/tests/e2e/offline_inference/stage_configs/rocm/qwen3_omni_ci.yaml @@ -7,21 +7,21 @@ stage_args: - stage_id: 0 runtime: - devices: "0,1" + devices: "0" max_batch_size: 1 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" enable_prefix_caching: false hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 load_format: dummy final_output: true final_output_type: text @@ -44,7 +44,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -68,7 +68,7 @@ stage_args: - stage_id: 2 runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav diff --git a/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml index 4db11e11f..8f0161edd 100644 --- a/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml @@ -8,14 +8,14 @@ stage_args: - stage_id: 0 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0,1" + devices: "0" max_batch_size: 5 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output hidden states for talker @@ -23,7 +23,7 @@ stage_args: enable_prefix_caching: false max_num_batched_tokens: 32768 hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 load_format: dummy final_output: true final_output_type: text @@ -47,7 +47,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -72,7 +72,7 @@ stage_args: - stage_id: 2 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav diff --git a/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml b/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml index 3297347c4..59642a77b 100644 --- a/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml +++ b/tests/e2e/online_serving/stage_configs/rocm/qwen3_omni_ci.yaml @@ -6,21 +6,21 @@ stage_args: - stage_id: 0 runtime: - devices: "0,1" + devices: "0" max_batch_size: 5 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" enable_prefix_caching: false hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 final_output: true final_output_type: text is_comprehension: true @@ -42,7 +42,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -65,7 +65,7 @@ stage_args: - stage_id: 2 runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav diff --git a/tests/e2e/stage_configs/qwen3_omni_ci.yaml b/tests/e2e/stage_configs/qwen3_omni_ci.yaml index 148a949d8..479e4d6e9 100644 --- a/tests/e2e/stage_configs/qwen3_omni_ci.yaml +++ b/tests/e2e/stage_configs/qwen3_omni_ci.yaml @@ -7,14 +7,14 @@ stage_args: - stage_id: 0 runtime: - devices: "0,1" + devices: "0" max_batch_size: 5 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output hidden states for talker @@ -23,7 +23,7 @@ stage_args: max_model_len: 32768 enable_prefix_caching: false hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 final_output: true final_output_type: text is_comprehension: true @@ -46,7 +46,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -68,7 +68,7 @@ stage_args: - stage_id: 2 runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml index d0380a39d..e3dcf940f 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml @@ -9,14 +9,14 @@ stage_args: - stage_id: 0 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0,1" + devices: "0" max_batch_size: 64 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output hidden states for talker @@ -24,7 +24,7 @@ stage_args: enable_prefix_caching: false max_num_batched_tokens: 32768 hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 final_output: true final_output_type: text is_comprehension: true @@ -47,7 +47,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -71,7 +71,7 @@ stage_args: - stage_id: 2 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml index 42748960e..1a8b60aa1 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml @@ -9,14 +9,14 @@ stage_args: - stage_id: 0 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0,1" + devices: "0" max_batch_size: 64 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output hidden states for talker @@ -24,7 +24,7 @@ stage_args: enable_prefix_caching: false max_num_batched_tokens: 32768 hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk final_output: true final_output_type: text @@ -41,14 +41,14 @@ stage_args: - stage_id: 1 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0" + devices: "1" max_batch_size: 64 engine_args: model_stage: talker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: false trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml index 69102df6f..0bc6e48e5 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_multiconnector.yaml @@ -8,21 +8,21 @@ stage_args: - stage_id: 0 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0,1" + devices: "0" max_batch_size: 1 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output hidden states for talker distributed_executor_backend: "mp" enable_prefix_caching: false hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 final_output: true final_output_type: text is_comprehension: true @@ -48,7 +48,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -77,7 +77,7 @@ stage_args: - stage_id: 2 stage_type: llm # Use llm stage type to launch OmniLLM runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav diff --git a/vllm_omni/platforms/rocm/stage_configs/qwen3_omni_moe.yaml b/vllm_omni/platforms/rocm/stage_configs/qwen3_omni_moe.yaml index 93e05ea9c..31312673a 100644 --- a/vllm_omni/platforms/rocm/stage_configs/qwen3_omni_moe.yaml +++ b/vllm_omni/platforms/rocm/stage_configs/qwen3_omni_moe.yaml @@ -7,14 +7,14 @@ stage_args: - stage_id: 0 runtime: - devices: "0,1" + devices: "0" max_batch_size: 1 engine_args: model_stage: thinker model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.6 + gpu_memory_utilization: 0.9 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output hidden states for talker @@ -22,7 +22,7 @@ stage_args: enable_prefix_caching: false max_num_batched_tokens: 32768 hf_config_name: thinker_config - tensor_parallel_size: 2 + tensor_parallel_size: 1 final_output: true final_output_type: text is_comprehension: true @@ -44,7 +44,7 @@ stage_args: model_arch: Qwen3OmniMoeForConditionalGeneration worker_type: ar scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler - gpu_memory_utilization: 0.3 + gpu_memory_utilization: 0.6 enforce_eager: true trust_remote_code: true engine_output_type: latent # Output codec codes for code2wav @@ -68,7 +68,7 @@ stage_args: - stage_id: 2 runtime: - devices: "0" + devices: "1" max_batch_size: 1 engine_args: model_stage: code2wav From f8dd566abda9cfa482e0a024aecee896123fe36e Mon Sep 17 00:00:00 2001 From: dsinghvi Date: Thu, 29 Jan 2026 18:50:39 +0530 Subject: [PATCH 60/83] [BugFix] Fix to use correct attn backend (#1038) Signed-off-by: Divyansh Singhvi --- vllm_omni/diffusion/attention/layer.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/vllm_omni/diffusion/attention/layer.py b/vllm_omni/diffusion/attention/layer.py index 0d3bf8b7d..83cfe84d7 100644 --- a/vllm_omni/diffusion/attention/layer.py +++ b/vllm_omni/diffusion/attention/layer.py @@ -12,6 +12,7 @@ import torch.nn as nn from vllm.logger import init_logger from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata +from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend from vllm_omni.diffusion.attention.parallel import build_parallel_attention_strategy from vllm_omni.diffusion.attention.parallel.ring import RingParallelAttention from vllm_omni.diffusion.attention.selector import get_attn_backend @@ -45,6 +46,14 @@ class Attention(nn.Module): causal=causal, num_kv_heads=num_kv_heads, ) + # Instantiate fallback backend for float32 support + self.sdpa_fallback = SDPABackend.get_impl_cls()( + num_heads=num_heads, + head_size=head_size, + softmax_scale=softmax_scale, + causal=causal, + num_kv_heads=num_kv_heads, + ) self.backend_pref = None self.softmax_scale = softmax_scale @@ -104,12 +113,12 @@ class Attention(nn.Module): return out def _run_local_attention(self, query, key, value, attn_metadata): - if self.backend_pref == "flash_attn" and query.dtype == torch.float32: - logger.warning( - "Flash Attention does not support float32. Overriding user config " + if query.dtype == torch.float32: + logger.warning_once( + f"Only SDPA supports float32. Overriding user config {type(self.attention)} " f"attention_backend='{self.backend_pref}' to 'sdpa' for dtype={query.dtype}." ) - self.backend_pref = "sdpa" + return self.sdpa_fallback.forward(query, key, value, attn_metadata) # Fallback to standard attention return self.attention.forward(query, key, value, attn_metadata) From a909d364153ab98da3863b39f1201822469a4751 Mon Sep 17 00:00:00 2001 From: Ziming Huang Date: Thu, 29 Jan 2026 21:23:19 +0800 Subject: [PATCH 61/83] [Perf] Qwen3 Omni talker mtp optimization (#1005) Signed-off-by: ZeldaHuang Signed-off-by: Hongsheng Liu Co-authored-by: Hongsheng Liu --- .../qwen3_omni_moe_code_predictor_mtp.py | 323 ++++++++---------- .../qwen3_omni/qwen3_omni_moe_talker.py | 62 +--- vllm_omni/worker/gpu_model_runner.py | 12 +- 3 files changed, 157 insertions(+), 240 deletions(-) diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py index bcefe13e8..6c135926d 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py @@ -13,9 +13,15 @@ import torch import torch.nn as nn import torch.nn.functional as F from transformers import Cache, PretrainedConfig -from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import Qwen3OmniMoeRotaryEmbedding +from transformers.generation.logits_process import ( + LogitsProcessorList, + TopKLogitsWarper, + TopPLogitsWarper, +) +from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, ModelConfig, VllmConfig +from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config +from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -24,81 +30,12 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding +from vllm.utils.torch_utils import direct_register_custom_op logger = init_logger(__name__) - -# ============================================================================ -# Rotary Embeddings and Helper Functions -# ============================================================================ - - -class Qwen3OmniCodePredictorRotaryEmbedding(nn.Module): - """Rotary positional embeddings for the code predictor.""" - - def __init__(self, dim, max_position_embeddings=2048, base=10000): - super().__init__() - self.dim = dim - self.max_position_embeddings = max_position_embeddings - self.base = base - - inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float() / self.dim)) - self.register_buffer("inv_freq", inv_freq, persistent=False) - - def forward(self, x: torch.Tensor, position_ids: torch.LongTensor) -> tuple[torch.Tensor, torch.Tensor]: - inv_freq = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) - position = position_ids[:, None, :].float() - freqs = (inv_freq @ position).transpose(1, 2) - emb = torch.cat((freqs, freqs), dim=-1) - cos = emb.cos().to(x.dtype) - sin = emb.sin().to(x.dtype) - return cos, sin - - -def rotate_half(x): - """Rotates half the hidden dims of the input.""" - x1 = x[..., : x.shape[-1] // 2] - x2 = x[..., x.shape[-1] // 2 :] - return torch.cat((-x2, x1), dim=-1) - - -def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): - """Applies Rotary Position Embedding to the query and key tensors. - - Args: - q (`torch.Tensor`): The query tensor. - k (`torch.Tensor`): The key tensor. - cos (`torch.Tensor`): The cosine part of the rotary embedding. - sin (`torch.Tensor`): The sine part of the rotary embedding. - position_ids (`torch.Tensor`, *optional*): - Deprecated and unused. - unsqueeze_dim (`int`, *optional*, defaults to 1): - The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - Returns: - `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. - """ - cos = cos.unsqueeze(unsqueeze_dim) - sin = sin.unsqueeze(unsqueeze_dim) - q_embed = (q * cos) + (rotate_half(q) * sin) - k_embed = (k * cos) + (rotate_half(k) * sin) - return q_embed, k_embed - - -def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: - """Repeat key/value heads for grouped query attention.""" - if n_rep == 1: - return hidden_states - bsz, num_kv_heads, seq_len, head_dim = hidden_states.shape - hidden_states = hidden_states[:, :, None, :, :].expand(bsz, num_kv_heads, n_rep, seq_len, head_dim) - return hidden_states.reshape(bsz, num_kv_heads * n_rep, seq_len, head_dim) - - # ============================================================================ # Code Predictor Attention Layer # ============================================================================ @@ -149,6 +86,12 @@ class Qwen3OmniCodePredictorAttention(nn.Module): prefix=f"{prefix}.o_proj", disable_tp=True, ) + self.rotary_emb = get_rope( + self.head_dim, + max_position=config.code_predictor_config.max_position_embeddings, + rope_parameters=None, + dual_chunk_attention_config=None, + ) self.q_size = self.num_heads * self.head_dim self.kv_size = self.num_key_value_heads * self.head_dim @@ -159,12 +102,19 @@ class Qwen3OmniCodePredictorAttention(nn.Module): self.is_causal = True self.config = config + self.attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"] + cudagraph_mode = get_current_vllm_config().compilation_config.cudagraph_mode + if "flash_attention_2" in ALL_ATTENTION_FUNCTIONS and cudagraph_mode.has_full_cudagraphs(): + logger.warning( + f"CUDAGraphMode.{cudagraph_mode.name} is currently not supported " + f"with flash attention for Qwen3-Omni talker MTP." + f"removing flash attention from attention_backends" + ) + self.attention_backends.remove("flash_attention_2") + def forward( self, hidden_states: torch.Tensor, - causal_mask: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, past_key_values: Cache | None = None, cache_position: torch.LongTensor | None = None, use_cache: bool = False, @@ -176,34 +126,39 @@ class Qwen3OmniCodePredictorAttention(nn.Module): q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) # Reshape for attention - q = q.view(bsz, seq_len, self.num_heads, self.head_dim) - k = k.view(bsz, seq_len, self.num_key_value_heads, self.head_dim) - v = v.view(bsz, seq_len, self.num_key_value_heads, self.head_dim) + q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim) + k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim) + v = v.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim) # Apply normalization - q = self.q_norm(q) - k = self.k_norm(k) + q = self.q_norm(q).contiguous() + k = self.k_norm(k).contiguous() + q = q.reshape(-1, self.q_size) + k = k.reshape(-1, self.kv_size) # Apply RoPE + q, k = self.rotary_emb(position_ids, q, k) + + # Reshape for attention + q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim) + k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim) + + v_heads = v.transpose(1, 2).contiguous() q_heads = q.transpose(1, 2).contiguous() k_heads = k.transpose(1, 2).contiguous() - q_heads, k_heads = apply_rotary_pos_emb(q_heads, k_heads, cos, sin) - v_heads = v.transpose(1, 2).contiguous() if past_key_values is not None: + sin, cos = self.rotary_emb.get_cos_sin(seq_len) # sin and cos are specific to RoPE models; cache_position needed for the static cache cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} k_heads, v_heads = past_key_values.update(k_heads, v_heads, self.layer_idx, cache_kwargs) - from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS - # Try attention backends in order of preference, with runtime error handling # This handles cases where the backend is registered but not actually available - attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"] attn_output = None last_error = None - for backend_name in attention_backends: + for backend_name in self.attention_backends: if backend_name not in ALL_ATTENTION_FUNCTIONS: continue @@ -219,15 +174,10 @@ class Qwen3OmniCodePredictorAttention(nn.Module): scaling=self.head_dim**-0.5, sliding_window=None, use_cache=use_cache, - position_ids=position_ids, + position_ids=position_ids[:seq_len].unsqueeze(0), output_hidden_states=True, output_attentions=False, ) - # Success - log fallback if not using flash_attention_2 - if backend_name != "flash_attention_2": - logger.warning_once( - f"Using {backend_name} attention backend (flash_attention_2 not available or failed)" - ) break except (ValueError, ImportError, RuntimeError, AttributeError) as e: # Store error and try next backend @@ -335,9 +285,6 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module): def mtp_block( self, hidden_states: torch.Tensor, - causal_mask: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, past_key_values: Cache | None = None, cache_position: torch.LongTensor | None = None, use_cache: bool = False, @@ -346,9 +293,7 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module): # Self-attention with residual residual = hidden_states hidden_states = self.input_layernorm(hidden_states) - hidden_states = self.self_attn( - hidden_states, causal_mask, cos, sin, past_key_values, cache_position, use_cache, position_ids - ) + hidden_states = self.self_attn(hidden_states, past_key_values, cache_position, use_cache, position_ids) hidden_states = residual + hidden_states # MLP with residual @@ -359,47 +304,6 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module): return hidden_states - def forward( - self, - input_ids: torch.Tensor, - positions: torch.Tensor, - previous_hidden_states: torch.Tensor, - inputs_embeds: torch.Tensor | None = None, - spec_step_index: int = 0, - ) -> torch.Tensor: - assert inputs_embeds is not None, "inputs_embeds required for MTP" - - # Mask position 0 (not needed for MTP) - inputs_embeds[positions == 0] = 0 - - hidden_states = torch.cat([inputs_embeds, previous_hidden_states], dim=-1) - - # Get position info for RoPE - batch_size, seq_len, _ = hidden_states.shape - device = hidden_states.device - position_ids = torch.arange(seq_len, dtype=torch.long, device=device).unsqueeze(0).expand(batch_size, -1) - - # Get RoPE embeddings - head_dim = self.self_attn.head_dim - rotary_emb = Qwen3OmniCodePredictorRotaryEmbedding( - head_dim, max_position_embeddings=self.config.code_predictor_config.max_position_embeddings - ) - rotary_emb = Qwen3OmniMoeRotaryEmbedding(self.config) - cos, sin = rotary_emb(hidden_states, position_ids) - - # Create causal mask - causal_mask = ( - torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool, device=device), diagonal=1) - .unsqueeze(0) - .unsqueeze(0) - ) - causal_mask = causal_mask.masked_fill(causal_mask, float("-inf")) - - # Forward through MTP block - hidden_states = self.mtp_block(hidden_states, causal_mask, cos, sin) - - return hidden_states - class Qwen3OmniCodePredictorBaseModel(nn.Module): """ @@ -444,9 +348,6 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module): self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - # RoPE - self.rotary_emb = Qwen3OmniMoeRotaryEmbedding(config=config) - def forward( self, inputs_embeds: torch.Tensor, @@ -462,7 +363,6 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module): Args: inputs_embeds: [batch, seq_len, hidden_size] - attention_mask: Optional attention mask tensor position_ids: Optional position IDs tensor past_key_values: Optional cached key-value pairs use_cache: Whether to use cache @@ -473,37 +373,11 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module): Named tuple with .last_hidden_state and .past_key_values attributes """ batch_size, seq_len, _ = inputs_embeds.shape - - # Create positions tensor if not provided - # positions must be [num_tokens] or [batch_size, seq_len] - if position_ids is None: - if cache_position is not None: - position_ids = cache_position # [num_tokens] - else: - position_ids = torch.arange(seq_len, device=inputs_embeds.device).unsqueeze(0) # [1, seq_len] - else: - position_ids = position_ids.flatten() # Ensure [num_tokens] - - # Extract cos/sin from rotary_emb cache - # The cos_sin_cache is [max_pos, rotary_dim * 2] - cos, sin = self.rotary_emb(inputs_embeds, position_ids) - - # Create causal mask - device = inputs_embeds.device - causal_mask = ( - torch.triu(torch.ones(seq_len, seq_len, dtype=torch.bool, device=device), diagonal=1) - .unsqueeze(0) - .unsqueeze(0) - ) - causal_mask = causal_mask.masked_fill(causal_mask, float("-inf")) - # Forward through decoder layers hidden_states = inputs_embeds for layer in self.layers: - hidden_states = layer.mtp_block( - hidden_states, causal_mask, cos, sin, past_key_values, cache_position, use_cache, position_ids - ) + hidden_states = layer.mtp_block(hidden_states, past_key_values, cache_position, use_cache, position_ids) # Final norm hidden_states = self.norm(hidden_states) @@ -517,6 +391,32 @@ class Qwen3OmniCodePredictorBaseModel(nn.Module): return self.codec_embedding +def code_predictor_sample( + logits: torch.Tensor, + layer_name: str, +) -> torch.Tensor: + forward_context = get_forward_context() + self = forward_context.no_compile_layers[layer_name] + logits = self.logits_processors(None, logits[:, -1]) + probs = F.softmax(logits, dim=-1) + code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1] + return code + + +def code_predictor_sample_fake( + logits: torch.Tensor, + layer_name: str, +) -> torch.Tensor: + return torch.empty((logits.shape[0], 1), dtype=torch.int64, device=logits.device) + + +direct_register_custom_op( + op_name="qwen3_omni_code_predictor_sample", + op_func=code_predictor_sample, + fake_impl=code_predictor_sample_fake, +) + + @support_torch_compile class Qwen3OmniMoeTalkerCodePredictor(nn.Module): """ @@ -552,30 +452,89 @@ class Qwen3OmniMoeTalkerCodePredictor(nn.Module): for _ in range(self.num_code_groups - 1) ] ) + self.logits_processors = LogitsProcessorList( + [ + TopKLogitsWarper(top_k=50), + TopPLogitsWarper(top_p=0.8), + ] + ) + max_batch_size = vllm_config.scheduler_config.max_num_seqs + self.position_ids_buffer = [] + for layer_idx in range(self.num_code_groups - 1): + self.position_ids_buffer.append(torch.arange(layer_idx + 2, dtype=torch.int64).repeat(max_batch_size)) + + compilation_config = get_current_vllm_config().compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError(f"Duplicate layer name: {prefix}") + compilation_config.static_forward_context[prefix] = self + self.layer_name = prefix def forward( self, - inputs_embeds: torch.Tensor, # [batch, seq_len, hidden_size] - layer_idx: int, # Which layer to predict (0-num_layers-2 for layers 1-num_layers-1) + layer0_code: torch.Tensor, + layer0_embed: torch.Tensor, + last_talker_hidden: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: """ Forward pass for code predictor. Args: - inputs_embeds: Input embeddings [batch_size, seq_len, hidden_size] - layer_idx: Which residual layer to predict (0-num_layers-2 for layers 1-num_layers-1) + layer0_code: + Code index for code-group (layer) 0. + Shape: [batch_size, 1], dtype typically int64. + + last_talker_hidden: + + Shape: [batch_size, hidden_size]. Returns: - logits: Predicted logits [batch_size, seq_len, vocab_size] - hidden_states: Output hidden states [batch_size, seq_len, hidden_size] + pos_all_layers: + Predicted codes for all code groups, including `layer0_code`. + Shape: [batch_size, num_code_groups, 1]. + + current_input: + The final input embedding sequence after appending embeddings of all + predicted codes (one token per predicted layer). + Shape: [batch_size, num_code_groups + 2, hidden_size]. """ - # Pass through base model - hidden_states = self.model(inputs_embeds) + pos_codes = [layer0_code] # Start with layer 0: [batch, 1] + try: + current_input = torch.cat([last_talker_hidden, layer0_embed], dim=1) # [batch, 2, hidden_size] + except Exception as e: + print(f"Error in current_input: {e}") + print(f"last_talker_hidden shape: {last_talker_hidden.shape}") + print(f"prev_embed shape: {layer0_embed.shape}") + raise e + batch_size = current_input.shape[0] - # Get logits from corresponding head - logits = self.lm_head[layer_idx](hidden_states) + # Predict all residual layers (layers 1 to num_code_groups-1) autoregressively + begin_pos = 0 + for layer_idx in range(self.num_code_groups - 1): + # Input for this layer: [last_talker_hidden, prev_layer_embed] - return logits, hidden_states + # Forward through code_predictor model + position_ids = self.position_ids_buffer[layer_idx][: (layer_idx + 2) * batch_size] # [1, seq_len] + begin_pos += layer_idx + 2 + outputs = self.model( + inputs_embeds=current_input, + attention_mask=None, + position_ids=position_ids, + past_key_values=None, + use_cache=False, + cache_position=None, + ) + hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size] + + # Use the corresponding lm_head for this layer + logits = self.lm_head[layer_idx](hidden_state[:, -1:, :]) + code = torch.ops.vllm.qwen3_omni_code_predictor_sample(logits, self.layer_name) + pos_codes.append(code) + # Update prev_embed for next layer (if not last layer) + # layer_idx=0 predicts layer 1, embed with codec_embedding[1] + new_embed = self.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size] + current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size] + pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1] + return pos_all_layers, current_input def load_weights(self, weights: list[tuple[str, torch.Tensor]]) -> set[str]: """Load weights with mapping for fused QKV and gate_up projections. diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py index 5b032d86e..65cb43d0b 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py @@ -4,11 +4,6 @@ from typing import Any import torch import torch.nn as nn import torch.nn.functional as F -from transformers.generation.logits_process import ( - LogitsProcessorList, - TopKLogitsWarper, - TopPLogitsWarper, -) from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import ( Qwen3OmniMoeTalkerConfig, ) @@ -140,7 +135,10 @@ class Qwen3OmniMoeTalkerForConditionalGeneration( self.code_predictor = Qwen3OmniMoeTalkerCodePredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "code_predictor") ) - self.empty_code = torch.empty((1, 0), dtype=torch.long) + self.layer0_embed_buffer = torch.zeros( + (vllm_config.scheduler_config.max_num_seqs, 1, self.config.text_config.hidden_size), + dtype=vllm_config.model_config.dtype, + ) def code_predictor_forward( self, @@ -192,62 +190,18 @@ class Qwen3OmniMoeTalkerForConditionalGeneration( all_codes_per_position = [] middle_hidden_states = [] # Collect hidden states for each position - logits_processors = LogitsProcessorList( - [ - TopKLogitsWarper(top_k=top_k), - TopPLogitsWarper(top_p=top_p), - ] - ) # Generate residual layers for each position for pos in range(seq_len): layer0_code = input_ids[:, pos : pos + 1] # [batch, 1] - # Predict all residual layers (layers 1 to num_code_groups-1) autoregressively - pos_codes = [layer0_code] # Start with layer 0: [batch, 1] - # Initial input: [last_talker_hidden, layer0_embed] layer0_embed = self.embed_input_ids(layer0_code) - prev_embed = layer0_embed # Track previous layer embedding - try: - current_input = torch.cat([last_talker_hidden, prev_embed], dim=1) # [batch, 2, hidden_size] - except Exception as e: - print(f"Error in current_input: {e}") - print(f"last_talker_hidden shape: {last_talker_hidden.shape}") - print(f"prev_embed shape: {prev_embed.shape}") - raise e - - for layer_idx in range(self.num_code_groups - 1): - # Input for this layer: [last_talker_hidden, prev_layer_embed] - - # Forward through code_predictor model - outputs = self.code_predictor.model( - inputs_embeds=current_input, - attention_mask=None, - position_ids=None, - past_key_values=None, - use_cache=False, - cache_position=None, - ) - - hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size] - - # Use the corresponding lm_head for this layer - logits = self.code_predictor.lm_head[layer_idx](hidden_state[:, -1:, :]) # [batch, 1, vocab_size] - - logits = logits_processors(None, logits[:, -1]) - - # Sample from the filtered distribution - probs = F.softmax(logits, dim=-1) - code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1] - pos_codes.append(code) - - # Update prev_embed for next layer (if not last layer) - # layer_idx=0 predicts layer 1, embed with codec_embedding[1] - new_embed = self.code_predictor.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size] - current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size] + self.layer0_embed_buffer[:batch_size].copy_(layer0_embed) + pos_all_layers, current_input = self.code_predictor( + layer0_code, self.layer0_embed_buffer[:batch_size], last_talker_hidden + ) # Stack all layers for this position: [batch, num_code_groups, 1] - pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1] all_codes_per_position.append(pos_all_layers) middle_hidden_states.append(current_input[:, 2:-1, :]) diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index a486212aa..1e2beb046 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -643,12 +643,16 @@ class OmniGPUModelRunner(GPUModelRunner): ), ): if getattr(self.model, "talker", None) is not None and hasattr(self.model, "talker_mtp"): + num_tokens_padded_talker_mtp = num_tokens_padded + if num_tokens_padded_talker_mtp == self.max_num_tokens: + num_tokens_padded_talker_mtp = min(num_tokens_padded, self.max_num_reqs) outputs = self.talker_mtp( - self.talker_mtp_input_ids.gpu[:num_tokens_padded], - self.talker_mtp_inputs_embeds.gpu[:num_tokens_padded], - self.last_talker_hidden.gpu[:num_tokens_padded], - self.text_step.gpu[:num_tokens_padded], + self.talker_mtp_input_ids.gpu[:num_tokens_padded_talker_mtp], + self.talker_mtp_inputs_embeds.gpu[:num_tokens_padded_talker_mtp], + self.last_talker_hidden.gpu[:num_tokens_padded_talker_mtp], + self.text_step.gpu[:num_tokens_padded_talker_mtp], ) + self.compilation_config.cache_dir = None outputs = self.model( input_ids=input_ids, positions=positions, From c7f89ef0852a44ce378cc5f3ddafe782f3ffda66 Mon Sep 17 00:00:00 2001 From: Fanli Lin Date: Thu, 29 Jan 2026 22:02:24 +0800 Subject: [PATCH 62/83] [Wan2.2] Optimize memory usage with conditional transformer loading (#980) Signed-off-by: Lin, Fanli Signed-off-by: Samit <285365963@qq.com> Co-authored-by: Samit <285365963@qq.com> --- .../text_to_video/text_to_video.md | 2 +- .../text_to_video/text_to_video.py | 7 +- .../models/wan2_2/pipeline_wan2_2.py | 96 ++++++++++++++----- 3 files changed, 80 insertions(+), 25 deletions(-) diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index cd5bd2be4..b5d0b2adc 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -26,7 +26,7 @@ Key arguments: - `--num_frames`: Number of frames (Wan default is 81). - `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high). - `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string). -- `--boundary_ratio`: Boundary split ratio for low/high DiT. +- `--boundary_ratio`: Boundary split ratio for low/high DiT. Default `0.875` uses both transformers for best quality. Set to `1.0` to load only the low-noise transformer (saves noticeable memory with good quality, recommended if memory is limited). Set to `0.0` loads only the high-noise transformer (not recommended, lower quality). - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: path to save the generated video. - `--vae_use_slicing`: enable VAE slicing for memory optimization. diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 464ff9cca..b5c003dfb 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -32,7 +32,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--width", type=int, default=1280, help="Video width.") parser.add_argument("--num_frames", type=int, default=81, help="Number of frames (Wan default is 81).") parser.add_argument("--num_inference_steps", type=int, default=40, help="Sampling steps.") - parser.add_argument("--boundary_ratio", type=float, default=0.875, help="Boundary split ratio for low/high DiT.") + parser.add_argument( + "--boundary_ratio", + type=float, + default=0.875, + help="Boundary split ratio for low/high DiT. Default 0.875 uses both transformers for best quality. Set to 1.0 to load only the low-noise transformer (saves noticeable memory with good quality, recommended if memory is limited).", + ) parser.add_argument( "--flow_shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)." ) diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index 712c05400..562d8eec5 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -227,17 +227,30 @@ class Wan22Pipeline(nn.Module): except Exception: pass + self.boundary_ratio = od_config.boundary_ratio + + # Determine which transformers to load based on boundary_ratio + # boundary_ratio=1.0: only load transformer_2 (low-noise stage only) + # boundary_ratio=0.0: only load transformer (high-noise stage only) + # otherwise: load both transformers + load_transformer = self.boundary_ratio != 1.0 if self.boundary_ratio is not None else True + load_transformer_2 = self.has_transformer_2 and ( + self.boundary_ratio != 0.0 if self.boundary_ratio is not None else True + ) + # Set up weights sources for transformer(s) - self.weights_sources = [ - DiffusersPipelineLoader.ComponentSource( - model_or_path=od_config.model, - subfolder="transformer", - revision=None, - prefix="transformer.", - fall_back_to_pt=True, - ), - ] - if self.has_transformer_2: + self.weights_sources = [] + if load_transformer: + self.weights_sources.append( + DiffusersPipelineLoader.ComponentSource( + model_or_path=od_config.model, + subfolder="transformer", + revision=None, + prefix="transformer.", + fall_back_to_pt=True, + ) + ) + if load_transformer_2: self.weights_sources.append( DiffusersPipelineLoader.ComponentSource( model_or_path=od_config.model, @@ -257,14 +270,26 @@ class Wan22Pipeline(nn.Module): ).to(self.device) # Initialize transformers with correct config (weights loaded via load_weights) - transformer_config = load_transformer_config(model, "transformer", local_files_only) - self.transformer = create_transformer_from_config(transformer_config) - if self.has_transformer_2: + if load_transformer: + transformer_config = load_transformer_config(model, "transformer", local_files_only) + self.transformer = create_transformer_from_config(transformer_config) + else: + self.transformer = None + + if load_transformer_2: transformer_2_config = load_transformer_config(model, "transformer_2", local_files_only) self.transformer_2 = create_transformer_from_config(transformer_2_config) else: self.transformer_2 = None + # Store the active transformer config + if load_transformer: + self.transformer_config = self.transformer.config + elif load_transformer_2: + self.transformer_config = self.transformer_2.config + else: + raise RuntimeError("No transformer loaded") + # Initialize UniPC scheduler flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 # default for 720p self.scheduler = FlowUniPCMultistepScheduler( @@ -275,7 +300,6 @@ class Wan22Pipeline(nn.Module): self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4 self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8 - self.boundary_ratio = od_config.boundary_ratio self._guidance_scale = None self._guidance_scale_2 = None @@ -333,7 +357,7 @@ class Wan22Pipeline(nn.Module): # Ensure dimensions are compatible with VAE and patch size # For expand_timesteps mode, we need latent dims to be even (divisible by patch_size) - patch_size = self.transformer.config.patch_size + patch_size = self.transformer_config.patch_size mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V height = (height // mod_value) * mod_value width = (width // mod_value) * mod_value @@ -374,7 +398,14 @@ class Wan22Pipeline(nn.Module): num_frames = max(num_frames, 1) device = self.device - dtype = self.transformer.dtype + # Get dtype from whichever transformer is loaded + if self.transformer is not None: + dtype = self.transformer.dtype + elif self.transformer_2 is not None: + dtype = self.transformer_2.dtype + else: + # Fallback to text_encoder dtype if no transformer loaded + dtype = self.text_encoder.dtype # Seed / generator if generator is None: @@ -444,7 +475,7 @@ class Wan22Pipeline(nn.Module): image_tensor = image # Use out_channels for noise latents (not in_channels which includes condition) - num_channels_latents = self.transformer.config.out_channels + num_channels_latents = self.transformer_config.out_channels batch_size = prompt_embeds.shape[0] # Prepare noise latents @@ -489,7 +520,7 @@ class Wan22Pipeline(nn.Module): first_frame_mask[:, :, 0] = 0 else: # T2V mode: standard latent preparation - num_channels_latents = self.transformer.config.in_channels + num_channels_latents = self.transformer_config.in_channels latents = self.prepare_latents( batch_size=prompt_embeds.shape[0], num_channels_latents=num_channels_latents, @@ -508,11 +539,30 @@ class Wan22Pipeline(nn.Module): # Denoising for t in timesteps: self._current_timestep = t - current_model = self.transformer - current_guidance_scale = guidance_low - if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None: - current_model = self.transformer_2 + + # Select model based on timestep and boundary_ratio + # High noise stage (t >= boundary_timestep): use transformer + # Low noise stage (t < boundary_timestep): use transformer_2 + if boundary_timestep is not None and t < boundary_timestep: + # Low noise stage - always use guidance_high for this stage current_guidance_scale = guidance_high + if self.transformer_2 is not None: + current_model = self.transformer_2 + elif self.transformer is not None: + # Fallback to transformer if transformer_2 not loaded + current_model = self.transformer + else: + raise RuntimeError("No transformer available for low-noise stage") + else: + # High noise stage - always use guidance_low for this stage + current_guidance_scale = guidance_low + if self.transformer is not None: + current_model = self.transformer + elif self.transformer_2 is not None: + # Fallback to transformer_2 if transformer not loaded + current_model = self.transformer_2 + else: + raise RuntimeError("No transformer available for high-noise stage") if self.expand_timesteps and latent_condition is not None: # I2V mode: blend condition with latents using mask @@ -520,7 +570,7 @@ class Wan22Pipeline(nn.Module): latent_model_input = latent_model_input.to(dtype) # Expand timesteps per patch - use floor division to match patch embedding - patch_size = self.transformer.config.patch_size + patch_size = self.transformer_config.patch_size num_latent_frames = latents.shape[2] patch_height = latents.shape[3] // patch_size[1] patch_width = latents.shape[4] // patch_size[2] From bfbf3e5b69492b12e8349fa7f6b62145a57c58df Mon Sep 17 00:00:00 2001 From: Fanli Lin Date: Thu, 29 Jan 2026 23:33:05 +0800 Subject: [PATCH 63/83] [Feat] Support XPU Backend in vLLM-Omni (#191) Signed-off-by: Fanli Lin Signed-off-by: Fanli Lin Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- vllm_omni/diffusion/offload.py | 26 +++-- .../qwen2_5_omni/qwen2_5_omni_token2wav.py | 4 + vllm_omni/platforms/interface.py | 9 ++ .../xpu/stage_configs/qwen2_5_omni.yaml | 101 ++++++++++++++++++ .../xpu/stage_configs/qwen3_omni_moe.yaml | 99 +++++++++++++++++ vllm_omni/platforms/xpu/utils.py | 16 +++ vllm_omni/platforms/xpu/worker/__init__.py | 0 .../xpu/worker/xpu_ar_model_runner.py | 19 ++++ .../platforms/xpu/worker/xpu_ar_worker.py | 15 +++ .../xpu/worker/xpu_generation_model_runner.py | 19 ++++ .../xpu/worker/xpu_generation_worker.py | 15 +++ 11 files changed, 315 insertions(+), 8 deletions(-) create mode 100644 vllm_omni/platforms/xpu/stage_configs/qwen2_5_omni.yaml create mode 100644 vllm_omni/platforms/xpu/stage_configs/qwen3_omni_moe.yaml create mode 100644 vllm_omni/platforms/xpu/utils.py create mode 100644 vllm_omni/platforms/xpu/worker/__init__.py create mode 100644 vllm_omni/platforms/xpu/worker/xpu_ar_model_runner.py create mode 100644 vllm_omni/platforms/xpu/worker/xpu_ar_worker.py create mode 100644 vllm_omni/platforms/xpu/worker/xpu_generation_model_runner.py create mode 100644 vllm_omni/platforms/xpu/worker/xpu_generation_worker.py diff --git a/vllm_omni/diffusion/offload.py b/vllm_omni/diffusion/offload.py index 4220ee9ff..174c64342 100644 --- a/vllm_omni/diffusion/offload.py +++ b/vllm_omni/diffusion/offload.py @@ -18,6 +18,8 @@ import torch from torch import nn from vllm.logger import init_logger +from vllm_omni.platforms import current_omni_platform + if TYPE_CHECKING: from vllm_omni.diffusion.data import OmniDiffusionConfig @@ -63,8 +65,8 @@ class SequentialOffloader: module.to("cpu", non_blocking=True) # Release allocator blocks when tensors leave the GPU. - if previous_device.type == "cuda" and torch.cuda.is_available(): - torch.cuda.empty_cache() + if previous_device.type != "cpu": + current_omni_platform.empty_cache() if self.pin_memory: for p in module.parameters(): @@ -87,7 +89,9 @@ class SequentialOffloader: for enc in self.encoders: self._to_cpu(enc) self._to_gpu(module) - torch.cuda.synchronize() + + current_omni_platform.synchronize() + logger.debug("Swapped: encoders -> CPU, DiT -> GPU") def _encoder_pre_hook(self, module: nn.Module, args: tuple) -> None: @@ -95,7 +99,9 @@ class SequentialOffloader: for dit_mod in self.dits: self._to_cpu(dit_mod) self._to_gpu(module) - torch.cuda.synchronize() + + current_omni_platform.synchronize() + logger.debug("Swapped: DiT -> CPU, encoder -> GPU") def register(self) -> None: @@ -166,7 +172,10 @@ def apply_offload_hooks( try: device = next(dit_modules[0].parameters()).device except StopIteration: - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + try: + device = current_omni_platform.get_torch_device() + except (NotImplementedError, AttributeError): + device = torch.device("cpu") # Collect all encoders encoders: list[nn.Module] = [] @@ -184,9 +193,10 @@ def apply_offload_hooks( pin = getattr(od_config, "pin_cpu_memory", True) for dit_mod in dit_modules: dit_mod.to("cpu") - if torch.cuda.is_available(): - torch.cuda.empty_cache() - if pin and torch.cuda.is_available(): + + current_omni_platform.empty_cache() + + if pin: for dit_mod in dit_modules: for p in dit_mod.parameters(): if p.data.device.type == "cpu" and not p.data.is_pinned(): diff --git a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py index 9d176ac6c..84ff8b55f 100644 --- a/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py +++ b/vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_token2wav.py @@ -736,6 +736,10 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) -> kaiser_window = torch.kaiser_window( kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu" ).to("npu") + elif current_omni_platform.is_xpu(): + kaiser_window = torch.kaiser_window( + kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu" + ).to("xpu") else: kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32) diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py index 2aac4b7a1..db541ebce 100644 --- a/vllm_omni/platforms/interface.py +++ b/vllm_omni/platforms/interface.py @@ -31,6 +31,15 @@ class OmniPlatform(Platform): def is_npu(self) -> bool: return self._omni_enum == OmniPlatformEnum.NPU + def is_xpu(self) -> bool: + return self._omni_enum == OmniPlatformEnum.XPU + + def is_cuda(self) -> bool: + return self._omni_enum == OmniPlatformEnum.CUDA + + def is_rocm(self) -> bool: + return self._omni_enum == OmniPlatformEnum.ROCM + @classmethod def get_omni_ar_worker_cls(cls) -> str: raise NotImplementedError diff --git a/vllm_omni/platforms/xpu/stage_configs/qwen2_5_omni.yaml b/vllm_omni/platforms/xpu/stage_configs/qwen2_5_omni.yaml new file mode 100644 index 000000000..3fe1b79e4 --- /dev/null +++ b/vllm_omni/platforms/xpu/stage_configs/qwen2_5_omni.yaml @@ -0,0 +1,101 @@ +# stage config for running qwen2.5-omni with architecture of OmniLLM. + +# The following config has been verified on 2x 1550-64G XPUs. +stage_args: + - stage_id: 0 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + process: true # Run this stage in a separate process + devices: "0" # Visible devices for this stage + max_batch_size: 1 + engine_args: + model_stage: thinker + model_arch: Qwen2_5OmniForConditionalGeneration + worker_type: ar + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + gpu_memory_utilization: 0.8 + enforce_eager: false + trust_remote_code: true + engine_output_type: latent + enable_prefix_caching: false + is_comprehension: true + final_output: true + final_output_type: text + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 2048 + seed: 42 + detokenize: True + repetition_penalty: 1.1 + - stage_id: 1 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + process: true + devices: "1" + max_batch_size: 1 + engine_args: + model_stage: talker + model_arch: Qwen2_5OmniForConditionalGeneration + worker_type: ar + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + gpu_memory_utilization: 0.8 + enforce_eager: false + trust_remote_code: true + enable_prefix_caching: false + engine_output_type: latent + engine_input_source: [0] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker + default_sampling_params: + temperature: 0.9 + top_p: 0.8 + top_k: 40 + max_tokens: 2048 + seed: 42 + detokenize: True + repetition_penalty: 1.05 + stop_token_ids: [8294] + + - stage_id: 2 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + process: true + devices: "0" + max_batch_size: 1 + engine_args: + model_stage: code2wav + model_arch: Qwen2_5OmniForConditionalGeneration + worker_type: generation + scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler + gpu_memory_utilization: 0.15 + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + engine_output_type: audio + engine_input_source: [1] + final_output: true + final_output_type: audio + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 2048 + seed: 42 + detokenize: True + repetition_penalty: 1.1 + +# Top-level runtime config (concise): default windows and stage edges +runtime: + enabled: true + defaults: + window_size: -1 # Simplified: trigger downstream only after full upstream completion + max_inflight: 1 # Simplified: process serially within each stage + + edges: + - from: 0 # thinker → talker: trigger only after receiving full input (-1) + to: 1 + window_size: -1 + - from: 1 # talker → code2wav: trigger only after receiving full input (-1) + to: 2 + window_size: -1 diff --git a/vllm_omni/platforms/xpu/stage_configs/qwen3_omni_moe.yaml b/vllm_omni/platforms/xpu/stage_configs/qwen3_omni_moe.yaml new file mode 100644 index 000000000..0803d7353 --- /dev/null +++ b/vllm_omni/platforms/xpu/stage_configs/qwen3_omni_moe.yaml @@ -0,0 +1,99 @@ +# Stage config for running Qwen3-Omni-MoE with 3-stage architecture +# Stage 0: Thinker (multimodal understanding + text generation) +# Stage 1: Talker (text embeddings → 8-layer RVQ codec codes) +# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform) + +# The following config has been verified on 4x 1550-64G XPUs. +stage_args: + - stage_id: 0 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + devices: "0,1" + max_batch_size: 1 + engine_args: + model_stage: thinker + model_arch: Qwen3OmniMoeForConditionalGeneration + worker_type: ar + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + gpu_memory_utilization: 0.8 + enforce_eager: true + trust_remote_code: true + engine_output_type: latent # Output hidden states for talker + distributed_executor_backend: "mp" + enable_prefix_caching: false + max_num_batched_tokens: 32768 + hf_config_name: thinker_config + tensor_parallel_size: 2 + final_output: true + final_output_type: text + is_comprehension: true + default_sampling_params: + temperature: 0.4 + top_p: 0.9 + top_k: 1 + max_tokens: 2048 + seed: 42 + detokenize: True + repetition_penalty: 1.05 + + - stage_id: 1 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + devices: "2" + max_batch_size: 1 + engine_args: + model_stage: talker + model_arch: Qwen3OmniMoeForConditionalGeneration + worker_type: ar + scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler + gpu_memory_utilization: 0.3 + enforce_eager: true + trust_remote_code: true + engine_output_type: latent # Output codec codes for code2wav + enable_prefix_caching: false + max_num_batched_tokens: 32768 + distributed_executor_backend: "mp" + hf_config_name: talker_config + engine_input_source: [0] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker + # final_output: true + # final_output_type: text + default_sampling_params: + temperature: 0.9 + top_k: 50 + max_tokens: 4096 + seed: 42 + detokenize: False + repetition_penalty: 1.05 + stop_token_ids: [2150] + + - stage_id: 2 + stage_type: llm # Use llm stage type to launch OmniLLM + runtime: + devices: "3" + max_batch_size: 1 + engine_args: + model_stage: code2wav + model_arch: Qwen3OmniMoeForConditionalGeneration + worker_type: generation + scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler + enforce_eager: true + trust_remote_code: true + enable_prefix_caching: false + engine_output_type: audio # Final output: audio waveform + gpu_memory_utilization: 0.1 + distributed_executor_backend: "mp" + max_num_batched_tokens: 1000000 + hf_config_name: thinker_config + engine_input_source: [1] + custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav + final_output: true + final_output_type: audio + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + seed: 42 + detokenize: True + repetition_penalty: 1.1 diff --git a/vllm_omni/platforms/xpu/utils.py b/vllm_omni/platforms/xpu/utils.py new file mode 100644 index 000000000..18437cae3 --- /dev/null +++ b/vllm_omni/platforms/xpu/utils.py @@ -0,0 +1,16 @@ +from contextlib import contextmanager + +import torch + + +@contextmanager +def torch_cuda_wrapper(): + try: + # replace cuda APIs with xpu APIs, this should work by default + torch.cuda.Stream = torch.xpu.Stream + torch.cuda.default_stream = torch.xpu.current_stream + torch.cuda.current_stream = torch.xpu.current_stream + torch.cuda.stream = torch.xpu.stream + yield + finally: + pass diff --git a/vllm_omni/platforms/xpu/worker/__init__.py b/vllm_omni/platforms/xpu/worker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/platforms/xpu/worker/xpu_ar_model_runner.py b/vllm_omni/platforms/xpu/worker/xpu_ar_model_runner.py new file mode 100644 index 000000000..2235d3548 --- /dev/null +++ b/vllm_omni/platforms/xpu/worker/xpu_ar_model_runner.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm_omni.platforms.xpu.utils import torch_cuda_wrapper +from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner + + +class XPUARModelRunner(GPUARModelRunner): + def __init__(self, *args, **kwargs): + with torch_cuda_wrapper(): + super().__init__(*args, **kwargs) + + def _init_device_properties(self): + self.num_sms = None + + def _sync_device(self) -> None: + torch.xpu.synchronize() diff --git a/vllm_omni/platforms/xpu/worker/xpu_ar_worker.py b/vllm_omni/platforms/xpu/worker/xpu_ar_worker.py new file mode 100644 index 000000000..ea6349264 --- /dev/null +++ b/vllm_omni/platforms/xpu/worker/xpu_ar_worker.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.v1.worker.xpu_worker import XPUWorker + +from vllm_omni.platforms.xpu.worker.xpu_ar_model_runner import XPUARModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin + + +class XPUARWorker(OmniWorkerMixin, XPUWorker): + """XPU AR worker for thinker/talker stages in Omni model.""" + + def init_device(self): + super().init_device() + self.model_runner: XPUARModelRunner = XPUARModelRunner(self.vllm_config, self.device) diff --git a/vllm_omni/platforms/xpu/worker/xpu_generation_model_runner.py b/vllm_omni/platforms/xpu/worker/xpu_generation_model_runner.py new file mode 100644 index 000000000..e865ac5e0 --- /dev/null +++ b/vllm_omni/platforms/xpu/worker/xpu_generation_model_runner.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm_omni.platforms.xpu.utils import torch_cuda_wrapper +from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner + + +class XPUGenerationModelRunner(GPUGenerationModelRunner): + def __init__(self, *args, **kwargs): + with torch_cuda_wrapper(): + super().__init__(*args, **kwargs) + + def _init_device_properties(self): + self.num_sms = None + + def _sync_device(self) -> None: + torch.xpu.synchronize() diff --git a/vllm_omni/platforms/xpu/worker/xpu_generation_worker.py b/vllm_omni/platforms/xpu/worker/xpu_generation_worker.py new file mode 100644 index 000000000..9c6d530eb --- /dev/null +++ b/vllm_omni/platforms/xpu/worker/xpu_generation_worker.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.v1.worker.xpu_worker import XPUWorker + +from vllm_omni.platforms.xpu.worker.xpu_generation_model_runner import XPUGenerationModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin + + +class XPUGenerationWorker(OmniWorkerMixin, XPUWorker): + """XPU generation worker for the code2wav (non-AR waveform generation) stage in the Omni model.""" + + def init_device(self): + super().init_device() + self.model_runner: XPUGenerationModelRunner = XPUGenerationModelRunner(self.vllm_config, self.device) From 3eecd2057b018b3984f607141916552f9be54932 Mon Sep 17 00:00:00 2001 From: dongbo910220 <32610838+dongbo910220@users.noreply.github.com> Date: Fri, 30 Jan 2026 00:28:29 +0800 Subject: [PATCH 64/83] [Fix] stabilize diffusion images LoRA E2E across CI drift (#1075) Signed-off-by: dongbo910220 <1275604947@qq.com> --- .../test_images_generations_lora.py | 128 +++++------------- 1 file changed, 33 insertions(+), 95 deletions(-) diff --git a/tests/e2e/online_serving/test_images_generations_lora.py b/tests/e2e/online_serving/test_images_generations_lora.py index 11165c6d2..e912c420d 100644 --- a/tests/e2e/online_serving/test_images_generations_lora.py +++ b/tests/e2e/online_serving/test_images_generations_lora.py @@ -12,10 +12,6 @@ This validates: import base64 import json import os -import signal -import subprocess -import sys -import time from io import BytesIO from pathlib import Path @@ -25,7 +21,8 @@ import requests import torch from PIL import Image from safetensors.torch import save_file -from vllm.utils.network_utils import get_open_port + +from tests.conftest import OmniServer os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -37,87 +34,6 @@ SIZE = "256x256" SEED = 42 -class OmniServer: - """Omniserver for vLLM-Omni tests.""" - - def __init__( - self, - model: str, - serve_args: list[str], - *, - env_dict: dict[str, str] | None = None, - ) -> None: - self.model = model - self.serve_args = serve_args - self.env_dict = env_dict - self.proc: subprocess.Popen | None = None - self.host = "127.0.0.1" - self.port = get_open_port() - - def _start_server(self) -> None: - env = os.environ.copy() - env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" - if self.env_dict is not None: - env.update(self.env_dict) - - cmd = [ - sys.executable, - "-m", - "vllm_omni.entrypoints.cli.main", - "serve", - self.model, - "--omni", - "--host", - self.host, - "--port", - str(self.port), - ] + self.serve_args - - print(f"Launching OmniServer with: {' '.join(cmd)}") - self.proc = subprocess.Popen( - cmd, - env=env, - cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), # vllm-omni root - start_new_session=True, - ) - - # Wait for server to be ready. - max_wait = 1200 - url = f"http://{self.host}:{self.port}/v1/models" - start_time = time.time() - while time.time() - start_time < max_wait: - try: - resp = requests.get(url, headers={"Authorization": "Bearer EMPTY"}, timeout=10) - if resp.status_code == 200: - print(f"Server ready on {self.host}:{self.port}") - return - except Exception: - pass - time.sleep(2) - - raise RuntimeError(f"Server failed to become ready within {max_wait} seconds") - - def __enter__(self): - self._start_server() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.proc is None: - return - try: - os.killpg(self.proc.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - self.proc.wait(timeout=30) - except subprocess.TimeoutExpired: - try: - os.killpg(self.proc.pid, signal.SIGKILL) - except ProcessLookupError: - pass - self.proc.wait() - - @pytest.fixture(scope="module") def omni_server(): with OmniServer(MODEL, ["--num-gpus", "1"]) as server: @@ -183,17 +99,30 @@ def _image_blue_tail_slice(img: Image.Image) -> np.ndarray: return tail -def _assert_slice_close(actual: np.ndarray, expected: np.ndarray, *, label: str) -> None: +def _slice_diff_stats(actual: np.ndarray, expected: np.ndarray) -> tuple[float, float]: + diff = np.abs(actual - expected) + return float(diff.max()), float(diff.mean()) + + +def _assert_slice_close( + actual: np.ndarray, + expected: np.ndarray, + *, + label: str, + base_max: float, + base_mean: float, +) -> None: assert actual.shape == (3, 3) assert expected.shape == (3, 3) - diff = np.abs(actual - expected) - max_diff = float(diff.max()) - mean_diff = float(diff.mean()) + max_diff, mean_diff = _slice_diff_stats(actual, expected) # NOTE: Different attention backends / torch.compile can introduce small # floating-point drift that shows up as a few LSBs in uint8 pixels. Keep # the reset check tolerant but bounded to avoid flaky CI. - assert max_diff <= 5.0 and mean_diff <= 3.0, ( - f"{label} slice mismatch (max={max_diff:.1f}, mean={mean_diff:.1f}): {actual.tolist()}" + max_thresh = max(10.0, base_max + 4.0) + mean_thresh = max(6.0, base_mean + 4.0) + assert max_diff <= max_thresh and mean_diff <= mean_thresh, ( + f"{label} slice mismatch (max={max_diff:.1f} > {max_thresh:.1f} or " + f"mean={mean_diff:.1f} > {mean_thresh:.1f}): {actual.tolist()}" ) @@ -219,6 +148,9 @@ def test_images_generations_per_request_lora_switching(omni_server: OmniServer, # Base generation. base_img = _post_images(omni_server, _basic_payload()) base_slice = _image_blue_tail_slice(base_img) + base_ref_img = _post_images(omni_server, _basic_payload()) + base_ref_slice = _image_blue_tail_slice(base_ref_img) + base_ref_max, base_ref_mean = _slice_diff_stats(base_ref_slice, base_slice) # Adapter A: apply delta to V slice only. lora_a_dir = tmp_path / "zimage_lora_a" @@ -245,11 +177,17 @@ def test_images_generations_per_request_lora_switching(omni_server: OmniServer, # Ensure switching back to no-LoRA restores the base output. base_img_2 = _post_images(omni_server, _basic_payload()) base_slice_2 = _image_blue_tail_slice(base_img_2) - _assert_slice_close(base_slice_2, base_slice, label="base_after_reset") - base_reset = float(np.abs(base_slice_2 - base_slice).mean()) + _, base_reset_mean = _slice_diff_stats(base_slice_2, base_slice) + _assert_slice_close( + base_slice_2, + base_slice, + label="base_after_reset", + base_max=base_ref_max, + base_mean=base_ref_mean, + ) # Ensure LoRA effects are clearly above the baseline drift. - min_delta = base_reset + 0.5 + min_delta = max(base_reset_mean + 1.0, 1.5) assert a_vs_base > min_delta, f"lora_a_vs_base drift too small: {a_vs_base} <= {min_delta}" assert b_vs_base > min_delta, f"lora_b_vs_base drift too small: {b_vs_base} <= {min_delta}" assert b_vs_a > min_delta, f"lora_b_vs_lora_a drift too small: {b_vs_a} <= {min_delta}" From 2c4974c6eabf5616c99eb7ab2e52da48ef0fa9d4 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Fri, 30 Jan 2026 00:55:53 +0800 Subject: [PATCH 65/83] [Bugfix][Test] Re-enable the log simple tests (#1065) Signed-off-by: gcanlin --- .buildkite/pipeline.yml | 2 ++ .buildkite/scripts/simple_test.sh | 58 ------------------------------- 2 files changed, 2 insertions(+), 58 deletions(-) delete mode 100755 .buildkite/scripts/simple_test.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 78a400cd8..6d731f127 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -22,8 +22,10 @@ steps: commands: - pytest -v -s tests/entrypoints/ - pytest -v -s tests/diffusion/cache/ + - pytest -v -s tests/diffusion/lora/ - pytest -v -s tests/model_executor/models/qwen2_5_omni/test_audio_length.py - pytest -v -s tests/worker/ + - pytest -v -s tests/distributed/omni_connectors/test_kv_flow.py agents: queue: "gpu_1_queue" plugins: diff --git a/.buildkite/scripts/simple_test.sh b/.buildkite/scripts/simple_test.sh deleted file mode 100755 index e76a5c70d..000000000 --- a/.buildkite/scripts/simple_test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)" -UV_BIN="uv" -PYTHON_BOOTSTRAP="${PYTHON:-python3}" -VENV_DIR="${VENV_DIR:-${REPO_ROOT}/.venv-simple-test}" -TARGET_PY_VERSION="3.12" - -if ! command -v "${PYTHON_BOOTSTRAP}" >/dev/null 2>&1; then - PYTHON_BOOTSTRAP="python" -fi - -cd "${REPO_ROOT}" - -# Ensure pip-installed scripts (including uv) are on PATH -PYTHON_SCRIPTS="$(${PYTHON_BOOTSTRAP} -c 'import sysconfig; print(sysconfig.get_path("scripts"))')" -USER_BASE="$(${PYTHON_BOOTSTRAP} -m site --user-base 2>/dev/null || true)" -PATH="${PYTHON_SCRIPTS}:${PATH}" -if [[ -n "${USER_BASE}" ]]; then - PATH="${USER_BASE}/bin:${PATH}" -fi - -if ! command -v "${UV_BIN}" >/dev/null 2>&1; then - "${PYTHON_BOOTSTRAP}" -m pip install --upgrade pip - "${PYTHON_BOOTSTRAP}" -m pip install uv - hash -r -fi - -TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)" -if [[ -z "${TARGET_PY_PATH}" ]]; then - "${UV_BIN}" python install "${TARGET_PY_VERSION}" - TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)" -fi - -if [[ -z "${TARGET_PY_PATH}" ]]; then - echo "Failed to locate python ${TARGET_PY_VERSION}" >&2 - exit 1 -fi - -if [[ ! -d "${VENV_DIR}" ]]; then - "${UV_BIN}" venv --python "${TARGET_PY_PATH}" "${VENV_DIR}" -fi - -VENV_PYTHON="${VENV_DIR}/bin/python" -[[ -x "${VENV_PYTHON}" ]] || { echo "Python not found in ${VENV_DIR}"; exit 1; } - -"${UV_BIN}" pip install --python "${VENV_PYTHON}" vllm==0.14.0 -"${UV_BIN}" pip install --python "${VENV_PYTHON}" -e ".[dev]" - -"${VENV_PYTHON}" -m pytest -v -s tests/entrypoints/ -"${VENV_PYTHON}" -m pytest -v -s tests/diffusion/cache/ -"${VENV_PYTHON}" -m pytest -v -s tests/diffusion/lora/ -"${VENV_PYTHON}" -m pytest -v -s tests/model_executor/models/qwen2_5_omni/test_audio_length.py -"${VENV_PYTHON}" -m pytest -v -s tests/worker/ -"${VENV_PYTHON}" -m pytest -v -s tests/distributed/omni_connectors/test_kv_flow.py From 47a0969976ae8dac9d2c567c2663720370a64df9 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Fri, 30 Jan 2026 01:09:34 +0800 Subject: [PATCH 66/83] [Bugfix] pr conflict fix, bugfix ignore mm data from stages to async omni (#1025) Signed-off-by: dengyunyang <584797741@qq.com> --- vllm_omni/entrypoints/omni_stage.py | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/vllm_omni/entrypoints/omni_stage.py b/vllm_omni/entrypoints/omni_stage.py index 2adb1c342..fc01c147c 100644 --- a/vllm_omni/entrypoints/omni_stage.py +++ b/vllm_omni/entrypoints/omni_stage.py @@ -1467,7 +1467,7 @@ async def _stage_worker_async( batch_request_ids, batch_request_outputs, _gen_ms_list, batch_metrics ): try: - r_outputs = [output] + r_outputs = [output_strip(output, omni_stage)] use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes) if use_shm: out_q.put( @@ -1553,3 +1553,32 @@ def make_stage_stats(_agg_total_tokens: int, _agg_total_gen_time_ms: float): from vllm_omni.entrypoints.log_utils import StageStats return StageStats(total_token=_agg_total_tokens, total_gen_time=_agg_total_gen_time_ms) + + +def output_strip(r_output: RequestOutput | OmniRequestOutput, omni_stage: OmniStage): + """ + Strip unnecessary multimodal outputs from stages results, + in order to: + - reduce memory usage + - reduce transfer & serialization overhead + """ + + # check multimodal data is required by stage output config. + if omni_stage.final_output and omni_stage.final_output_type != "text": + return r_output + + # If the request has already finished, should not be altered. + if getattr(r_output, "finished", False): + return r_output + + mm_output = getattr(r_output, "multimodal_output", None) + if mm_output is not None: + r_output.multimodal_output = {} + + outputs = getattr(r_output, "outputs", None) + if outputs is not None: + for out in outputs: + if getattr(out, "multimodal_output", None): + out.multimodal_output = {} + + return r_output From 0e07eb6ff920937e003399cd9a92b6055c37796d Mon Sep 17 00:00:00 2001 From: Ding Zuhao Date: Fri, 30 Jan 2026 07:56:10 +0800 Subject: [PATCH 67/83] [Doc][Bagel] Add BAGEL-7B-MoT documentation and edit the default stage configuration (#987) Signed-off-by: Ding Zuhao Signed-off-by: jzz --- .../examples/offline_inference/bagel.md | 179 ++++++++++++++ .../examples/online_serving/bagel.md | 232 ++++++++++++++++++ examples/offline_inference/bagel/README.md | 177 +++++++++++++ .../bagel/run_single_prompt.sh | 6 - examples/online_serving/bagel/README.md | 230 +++++++++++++++++ .../model_executor/stage_configs/bagel.yaml | 4 +- .../stage_configs/bagel_multiconnector.yaml | 2 +- 7 files changed, 821 insertions(+), 9 deletions(-) create mode 100644 docs/user_guide/examples/offline_inference/bagel.md create mode 100644 docs/user_guide/examples/online_serving/bagel.md create mode 100644 examples/offline_inference/bagel/README.md delete mode 100644 examples/offline_inference/bagel/run_single_prompt.sh create mode 100644 examples/online_serving/bagel/README.md diff --git a/docs/user_guide/examples/offline_inference/bagel.md b/docs/user_guide/examples/offline_inference/bagel.md new file mode 100644 index 000000000..b2ee5d241 --- /dev/null +++ b/docs/user_guide/examples/offline_inference/bagel.md @@ -0,0 +1,179 @@ +# BAGEL-7B-MoT + +Source . + +## Set up + +Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. + +## Run examples + +**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices. + +Get into the bagel folder + +```bash +cd examples/offline_inference/bagel +``` + +### Modality Control + +BAGEL-7B-MoT supports multiple modality modes. You can control the mode using the `--modality` argument: + +#### Text to Image (text2img) + +- **Pipeline**: Text → Thinker → DiT → VAE Decode → Image +- **Stages Used**: Stage 0 (Thinker) + Stage 1 (DiT) +- **KV Transfer**: Thinker sends KV cache to DiT for conditioned generation + +Generate images from text prompts: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2img \ + --prompts "A cute cat" +``` + +#### Image to Image (img2img) + +- **Pipeline**: Image → VAE Encode → DiT → VAE Decode → New Image +- **Stages Used**: Stage 1 (DiT) only +- **Special**: Bypasses the Thinker stage, direct image-to-image transformation + +Transform images based on text prompts: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality img2img \ + --image-path /path/to/image.jpg \ + --prompts "Let the woman wear a blue dress" +``` + +#### Image to Text (img2text) + +- **Pipeline**: Image → ViT + VAE Encode → Thinker → Text Output +- **Stages Used**: Stage 0 (Thinker) only +- **Special**: Uses both VAE latent encoding AND ViT semantic encoding for comprehensive image understanding + +Generate text descriptions from images: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality img2text \ + --image-path /path/to/image.jpg \ + --prompts "Describe this image in detail" +``` + +#### Text to Text (text2text) + +- **Pipeline**: Text → Thinker → Text Output +- **Stages Used**: Stage 0 (Thinker) only +- **Special**: No visual components involved, operates as pure language model + +Pure text generation: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2text \ + --prompts "What is the capital of France?" + +# You can load prompts from a text file (one prompt per line): +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2text \ + --txt-prompts /path/to/prompts.txt +``` + +### Inference Steps + +Control the number of inference steps for image generation: + +```bash +# You can adjust steps to 100 to improve image quality +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2img \ + --steps 50 \ + --prompts "A cute cat" +``` + +### Key arguments + +BAGEL-7B-MoT supports **multiple modality modes** for different use cases. + +The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml) + +#### 📌 Command Line Arguments (end2end.py) + +| Argument | Type | Default | Description | +| :--------------------- | :----- | :---------------------------- | :----------------------------------------------------------- | +| `--model` | string | `ByteDance-Seed/BAGEL-7B-MoT` | Model path or name | +| `--modality` | choice | `text2img` | Modality mode: `text2img`, `img2img`, `img2text`, `text2text` | +| `--prompts` | list | `None` | Input text prompts directly | +| `--txt-prompts` | string | `None` | Path to txt file with one prompt per line | +| `--image-path` | string | `None` | Input image path (for `img2img`/`img2text`) | +| `--steps` | int | `50` | Number of inference steps | +| `--stage-configs-path` | string | `None` | Custom stage config file path | +| `--worker-backend` | choice | `process` | Worker backend: `process` or `ray` | +| `--ray-address` | string | `None` | Ray cluster address | +| `--enable-stats` | flag | `False` | Enable statistics logging | +| `--init-sleep-seconds` | int | `20` | Initialization sleep time | +| `--batch-timeout` | int | `5` | Batch timeout | +| `--init-timeout` | int | `300` | Initialization timeout | + +------ + +#### ⚙️ Stage Configuration Parameters (bagel.yaml) + + **Stage 0 - Thinker (LLM Stage)** + +| Parameter | Value | Description | +| :------------------------------- | :------------------------------ | :----------------------- | +| `stage_type` | `llm` | Stage type | +| `devices` | `"0"` | GPU device ID | +| `max_batch_size` | `1` | Maximum batch size | +| `model_stage` | `thinker` | Model stage identifier | +| `model_arch` | `BagelForConditionalGeneration` | Model architecture | +| `gpu_memory_utilization` | `0.4` | GPU memory utilization | +| `tensor_parallel_size` | `1` | Tensor parallel size | +| `max_num_batched_tokens` | `32768` | Maximum batched tokens | +| `omni_kv_config.need_send_cache` | `true` | Whether to send KV cache | + +------ + +**Stage 1 - DiT (Diffusion Stage)** + +| Parameter | Value | Description | +| :------------------------------- | :---------- | :-------------------------- | +| `stage_type` | `diffusion` | Stage type | +| `devices` | `"0"` | GPU device ID | +| `max_batch_size` | `1` | Maximum batch size | +| `model_stage` | `dit` | Model stage identifier | +| `gpu_memory_utilization` | `0.4` | GPU memory utilization | +| `omni_kv_config.need_recv_cache` | `true` | Whether to receive KV cache | +| `engine_input_source` | `[0]` | Input source from Stage 0 | + +------ + +#### 🔗 Runtime Configuration + +| Parameter | Value | Description | +| :-------------------- | :------ | :------------------------------- | +| `window_size` | `-1` | Window size (-1 means unlimited) | +| `max_inflight` | `1` | Maximum inflight requests | +| `shm_threshold_bytes` | `65536` | Shared memory threshold (64KB) | + +## FAQ + +- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below. + +```bash +sudo apt update +sudo apt install ffmpeg +``` + +- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len. + +| Stage | VRAM | +| :------------------ | :--------------------------- | +| Stage-0 (Thinker) | **15.04 GiB** **+ KV Cache** | +| Stage-1 (DiT) | **26.50 GiB** | +| Total | **~42 GiB + KV Cache** | diff --git a/docs/user_guide/examples/online_serving/bagel.md b/docs/user_guide/examples/online_serving/bagel.md new file mode 100644 index 000000000..107e42449 --- /dev/null +++ b/docs/user_guide/examples/online_serving/bagel.md @@ -0,0 +1,232 @@ +# BAGEL-7B-MoT + +Source . + +## 🛠️ Installation + +Please refer to [README.md](../../../README.md) + +## Run examples (BAGEL-7B-MoT) + +**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices. + +### Launch the Server + +```bash +# Use default configuration +vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 +``` + +Or use the convenience script: + +```bash +cd /workspace/vllm-omni/examples/online_serving/bagel +bash run_server.sh +``` + +If you have a custom stage configs file, launch the server with the command below: + +```bash +vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file +``` + +### Send Multi-modal Request + +Get into the bagel folder: + +```bash +cd examples/online_serving/bagel +``` + +Send request via Python + +```bash +python openai_chat_client.py --prompt "A cute cat" --modality text2img +``` + +The Python client supports the following command-line arguments: + +- `--prompt` (or `-p`): Text prompt for generation (default: `A cute cat`) +- `--output` (or `-o`): Output file path for image results (default: `bagel_output.png`) +- `--server` (or `-s`): Server URL (default: `http://localhost:8091`) +- `--image-url` (or `-i`): Input image URL or local file path (for img2img/img2text modes) +- `--modality` (or `-m`): Task modality (default: `text2img`). Options: `text2img`, `img2img`, `img2text`, `text2text` +- `--height`: Image height in pixels (default: 512) +- `--width`: Image width in pixels (default: 512) +- `--steps`: Number of inference steps (default: 25) +- `--seed`: Random seed (default: 42) +- `--negative`: Negative prompt for image generation + +Example with custom parameters: + +```bash +python openai_chat_client.py \ + --prompt "A futuristic city" \ + --modality text2img \ + --height 768 \ + --width 768 \ + --steps 50 \ + --seed 42 \ + --negative "blurry, low quality" +``` + +## Modality Control + +BAGEL-7B-MoT supports **multiple modality modes** for different use cases. + +The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml) + +| Modality | Input | Output | Description | +| ----------- | ------------ | ------ | -------------------------------------- | +| `text2img` | Text | Image | Generate images from text prompts | +| `img2img` | Image + Text | Image | Transform images using text guidance | +| `img2text` | Image + Text | Text | Generate text descriptions from images | +| `text2text` | Text | Text | Pure text generation | + +### Text to Image (text2img) + +Generate images from text prompts: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "A beautiful sunset over mountains" \ + --modality text2img \ + --output sunset.png \ + --steps 50 +``` + +**Using curl** + +```bash +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>A beautiful sunset over mountains<|im_end|>"}]}], + "modalities": ["image"], + "height": 512, + "width": 512, + "num_inference_steps": 50, + "seed": 42 + }' +``` + + +### Image to Image (img2img) + +Transform images based on text prompts: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "Make the cat stand up" \ + --modality img2img \ + --image-url /path/to/input.jpg \ + --output transformed.png +``` + +**Using curl** + +```bash +IMAGE_BASE64=$(base64 -w 0 cat.jpg) + +cat < payload.json +{ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "<|im_start|>Make the cat stand up<|im_end|>"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}} + ] + }], + "modalities": ["image"], + "height": 512, + "width": 512, + "num_inference_steps": 50, + "seed": 42 +} +EOF + +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d @payload.json + +``` + +### Image to Text (img2text) + +Generate text descriptions from images: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "Describe this image in detail" \ + --modality img2text \ + --image-url /path/to/image.jpg +``` + +**Using curl** + +```bash +IMAGE_BASE64=$(base64 -w 0 cat.jpg) + +cat < payload.json +{ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "<|im_start|>user\n<|image_pad|>\nDescribe this image in detail<|im_end|>\n<|im_start|>assistant\n"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}} + ] + }], + "modalities": ["text"] +} +EOF + +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d @payload.json +``` + +### Text to Text (text2text) + +Pure text generation: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "What is the capital of France?" \ + --modality text2text +``` + +**Using curl** + +```bash +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n"}]}] + "modalities": ["text"] + }' +``` + +## FAQ + +- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below. + +```bash +sudo apt update +sudo apt install ffmpeg +``` + +- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len. + +| Stage | VRAM | +| :------------------ | :--------------------------- | +| Stage-0 (Thinker) | **15.04 GiB** **+ KV Cache** | +| Stage-1 (DiT) | **26.50 GiB** | +| Total | **~42 GiB + KV Cache** | diff --git a/examples/offline_inference/bagel/README.md b/examples/offline_inference/bagel/README.md new file mode 100644 index 000000000..7bcfb42fa --- /dev/null +++ b/examples/offline_inference/bagel/README.md @@ -0,0 +1,177 @@ +# BAGEL-7B-MoT + +## Set up + +Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. + +## Run examples + +**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices. + +Get into the bagel folder + +```bash +cd examples/offline_inference/bagel +``` + +### Modality Control + +BAGEL-7B-MoT supports multiple modality modes. You can control the mode using the `--modality` argument: + +#### Text to Image (text2img) + +- **Pipeline**: Text → Thinker → DiT → VAE Decode → Image +- **Stages Used**: Stage 0 (Thinker) + Stage 1 (DiT) +- **KV Transfer**: Thinker sends KV cache to DiT for conditioned generation + +Generate images from text prompts: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2img \ + --prompts "A cute cat" +``` + +#### Image to Image (img2img) + +- **Pipeline**: Image → VAE Encode → DiT → VAE Decode → New Image +- **Stages Used**: Stage 1 (DiT) only +- **Special**: Bypasses the Thinker stage, direct image-to-image transformation + +Transform images based on text prompts: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality img2img \ + --image-path /path/to/image.jpg \ + --prompts "Let the woman wear a blue dress" +``` + +#### Image to Text (img2text) + +- **Pipeline**: Image → ViT + VAE Encode → Thinker → Text Output +- **Stages Used**: Stage 0 (Thinker) only +- **Special**: Uses both VAE latent encoding AND ViT semantic encoding for comprehensive image understanding + +Generate text descriptions from images: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality img2text \ + --image-path /path/to/image.jpg \ + --prompts "Describe this image in detail" +``` + +#### Text to Text (text2text) + +- **Pipeline**: Text → Thinker → Text Output +- **Stages Used**: Stage 0 (Thinker) only +- **Special**: No visual components involved, operates as pure language model + +Pure text generation: + +```bash +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2text \ + --prompts "What is the capital of France?" + +# You can load prompts from a text file (one prompt per line): +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2text \ + --txt-prompts /path/to/prompts.txt +``` + +### Inference Steps + +Control the number of inference steps for image generation: + +```bash +# You can adjust steps to 100 to improve image quality +python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ + --modality text2img \ + --steps 50 \ + --prompts "A cute cat" +``` + +### Key arguments + +BAGEL-7B-MoT supports **multiple modality modes** for different use cases. + +The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml) + +#### 📌 Command Line Arguments (end2end.py) + +| Argument | Type | Default | Description | +| :--------------------- | :----- | :---------------------------- | :----------------------------------------------------------- | +| `--model` | string | `ByteDance-Seed/BAGEL-7B-MoT` | Model path or name | +| `--modality` | choice | `text2img` | Modality mode: `text2img`, `img2img`, `img2text`, `text2text` | +| `--prompts` | list | `None` | Input text prompts directly | +| `--txt-prompts` | string | `None` | Path to txt file with one prompt per line | +| `--image-path` | string | `None` | Input image path (for `img2img`/`img2text`) | +| `--steps` | int | `50` | Number of inference steps | +| `--stage-configs-path` | string | `None` | Custom stage config file path | +| `--worker-backend` | choice | `process` | Worker backend: `process` or `ray` | +| `--ray-address` | string | `None` | Ray cluster address | +| `--enable-stats` | flag | `False` | Enable statistics logging | +| `--init-sleep-seconds` | int | `20` | Initialization sleep time | +| `--batch-timeout` | int | `5` | Batch timeout | +| `--init-timeout` | int | `300` | Initialization timeout | + +------ + +#### ⚙️ Stage Configuration Parameters (bagel.yaml) + + **Stage 0 - Thinker (LLM Stage)** + +| Parameter | Value | Description | +| :------------------------------- | :------------------------------ | :----------------------- | +| `stage_type` | `llm` | Stage type | +| `devices` | `"0"` | GPU device ID | +| `max_batch_size` | `1` | Maximum batch size | +| `model_stage` | `thinker` | Model stage identifier | +| `model_arch` | `BagelForConditionalGeneration` | Model architecture | +| `gpu_memory_utilization` | `0.4` | GPU memory utilization | +| `tensor_parallel_size` | `1` | Tensor parallel size | +| `max_num_batched_tokens` | `32768` | Maximum batched tokens | +| `omni_kv_config.need_send_cache` | `true` | Whether to send KV cache | + +------ + +**Stage 1 - DiT (Diffusion Stage)** + +| Parameter | Value | Description | +| :------------------------------- | :---------- | :-------------------------- | +| `stage_type` | `diffusion` | Stage type | +| `devices` | `"0"` | GPU device ID | +| `max_batch_size` | `1` | Maximum batch size | +| `model_stage` | `dit` | Model stage identifier | +| `gpu_memory_utilization` | `0.4` | GPU memory utilization | +| `omni_kv_config.need_recv_cache` | `true` | Whether to receive KV cache | +| `engine_input_source` | `[0]` | Input source from Stage 0 | + +------ + +#### 🔗 Runtime Configuration + +| Parameter | Value | Description | +| :-------------------- | :------ | :------------------------------- | +| `window_size` | `-1` | Window size (-1 means unlimited) | +| `max_inflight` | `1` | Maximum inflight requests | +| `shm_threshold_bytes` | `65536` | Shared memory threshold (64KB) | + +## FAQ + +- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below. + +```bash +sudo apt update +sudo apt install ffmpeg +``` + +- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len. + +| Stage | VRAM | +| :------------------ | :--------------------------- | +| Stage-0 (Thinker) | **15.04 GiB** **+ KV Cache** | +| Stage-1 (DiT) | **26.50 GiB** | +| Total | **~42 GiB + KV Cache** | diff --git a/examples/offline_inference/bagel/run_single_prompt.sh b/examples/offline_inference/bagel/run_single_prompt.sh deleted file mode 100644 index 2099620e3..000000000 --- a/examples/offline_inference/bagel/run_single_prompt.sh +++ /dev/null @@ -1,6 +0,0 @@ -prompt="<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n" - -python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ - --prompt_type text \ - --init-sleep-seconds 0 \ - --prompts ${prompt} diff --git a/examples/online_serving/bagel/README.md b/examples/online_serving/bagel/README.md new file mode 100644 index 000000000..3fbea0550 --- /dev/null +++ b/examples/online_serving/bagel/README.md @@ -0,0 +1,230 @@ +# BAGEL-7B-MoT + +## 🛠️ Installation + +Please refer to [README.md](../../../README.md) + +## Run examples (BAGEL-7B-MoT) + +**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices. + +### Launch the Server + +```bash +# Use default configuration +vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 +``` + +Or use the convenience script: + +```bash +cd /workspace/vllm-omni/examples/online_serving/bagel +bash run_server.sh +``` + +If you have a custom stage configs file, launch the server with the command below: + +```bash +vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file +``` + +### Send Multi-modal Request + +Get into the bagel folder: + +```bash +cd examples/online_serving/bagel +``` + +Send request via Python + +```bash +python openai_chat_client.py --prompt "A cute cat" --modality text2img +``` + +The Python client supports the following command-line arguments: + +- `--prompt` (or `-p`): Text prompt for generation (default: `A cute cat`) +- `--output` (or `-o`): Output file path for image results (default: `bagel_output.png`) +- `--server` (or `-s`): Server URL (default: `http://localhost:8091`) +- `--image-url` (or `-i`): Input image URL or local file path (for img2img/img2text modes) +- `--modality` (or `-m`): Task modality (default: `text2img`). Options: `text2img`, `img2img`, `img2text`, `text2text` +- `--height`: Image height in pixels (default: 512) +- `--width`: Image width in pixels (default: 512) +- `--steps`: Number of inference steps (default: 25) +- `--seed`: Random seed (default: 42) +- `--negative`: Negative prompt for image generation + +Example with custom parameters: + +```bash +python openai_chat_client.py \ + --prompt "A futuristic city" \ + --modality text2img \ + --height 768 \ + --width 768 \ + --steps 50 \ + --seed 42 \ + --negative "blurry, low quality" +``` + +## Modality Control + +BAGEL-7B-MoT supports **multiple modality modes** for different use cases. + +The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml) + +| Modality | Input | Output | Description | +| ----------- | ------------ | ------ | -------------------------------------- | +| `text2img` | Text | Image | Generate images from text prompts | +| `img2img` | Image + Text | Image | Transform images using text guidance | +| `img2text` | Image + Text | Text | Generate text descriptions from images | +| `text2text` | Text | Text | Pure text generation | + +### Text to Image (text2img) + +Generate images from text prompts: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "A beautiful sunset over mountains" \ + --modality text2img \ + --output sunset.png \ + --steps 50 +``` + +**Using curl** + +```bash +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>A beautiful sunset over mountains<|im_end|>"}]}], + "modalities": ["image"], + "height": 512, + "width": 512, + "num_inference_steps": 50, + "seed": 42 + }' +``` + + +### Image to Image (img2img) + +Transform images based on text prompts: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "Make the cat stand up" \ + --modality img2img \ + --image-url /path/to/input.jpg \ + --output transformed.png +``` + +**Using curl** + +```bash +IMAGE_BASE64=$(base64 -w 0 cat.jpg) + +cat < payload.json +{ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "<|im_start|>Make the cat stand up<|im_end|>"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}} + ] + }], + "modalities": ["image"], + "height": 512, + "width": 512, + "num_inference_steps": 50, + "seed": 42 +} +EOF + +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d @payload.json + +``` + +### Image to Text (img2text) + +Generate text descriptions from images: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "Describe this image in detail" \ + --modality img2text \ + --image-url /path/to/image.jpg +``` + +**Using curl** + +```bash +IMAGE_BASE64=$(base64 -w 0 cat.jpg) + +cat < payload.json +{ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "<|im_start|>user\n<|image_pad|>\nDescribe this image in detail<|im_end|>\n<|im_start|>assistant\n"}, + {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}} + ] + }], + "modalities": ["text"] +} +EOF + +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d @payload.json +``` + +### Text to Text (text2text) + +Pure text generation: + +**Using Python client** + +```bash +python openai_chat_client.py \ + --prompt "What is the capital of France?" \ + --modality text2text +``` + +**Using curl** + +```bash +curl http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n"}]}] + "modalities": ["text"] + }' +``` + +## FAQ + +- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below. + +```bash +sudo apt update +sudo apt install ffmpeg +``` + +- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len. + +| Stage | VRAM | +| :------------------ | :--------------------------- | +| Stage-0 (Thinker) | **15.04 GiB** **+ KV Cache** | +| Stage-1 (DiT) | **26.50 GiB** | +| Total | **~42 GiB + KV Cache** | diff --git a/vllm_omni/model_executor/stage_configs/bagel.yaml b/vllm_omni/model_executor/stage_configs/bagel.yaml index 93d2b611a..ee2b17729 100644 --- a/vllm_omni/model_executor/stage_configs/bagel.yaml +++ b/vllm_omni/model_executor/stage_configs/bagel.yaml @@ -4,7 +4,7 @@ stage_args: - stage_id: 0 stage_type: llm runtime: - devices: "0" # Adjust as needed + devices: "0" max_batch_size: 1 engine_args: model_stage: thinker @@ -38,7 +38,7 @@ stage_args: - stage_id: 1 stage_type: diffusion runtime: - devices: "1" + devices: "0" max_batch_size: 1 engine_args: model_stage: dit diff --git a/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml b/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml index e378fe675..eddc844d5 100644 --- a/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml +++ b/vllm_omni/model_executor/stage_configs/bagel_multiconnector.yaml @@ -4,7 +4,7 @@ stage_args: - stage_id: 0 stage_type: llm runtime: - devices: "0" # Adjust as needed + devices: "0" max_batch_size: 1 engine_args: model_stage: thinker From c7fb59fd28c24235c6a7efb625d7a59b8f732bbf Mon Sep 17 00:00:00 2001 From: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:27:19 +0800 Subject: [PATCH 68/83] [Fix] Increase max wait time for server readiness to accommodate model loading (#1089) Signed-off-by: Andy Zhou <46011930+AndyZhou952@users.noreply.github.com> --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7737a7dc2..b1d7e6cc8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -920,7 +920,7 @@ class OmniServer: ) # Wait for server to be ready - max_wait = 600 # 10 minutes + max_wait = 1200 # 20 minutes start_time = time.time() while time.time() - start_time < max_wait: try: From 7bfd3c069d792aa6a0ee8ed3bea95a11dbf251be Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Fri, 30 Jan 2026 11:41:09 +0800 Subject: [PATCH 69/83] [Benchmark] Add vLLM-Omni Omni model online benchmark (#780) Signed-off-by: wangyu31577 Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: wangyu31577 Co-authored-by: Hongsheng Liu --- docs/cli/README.md | 18 + docs/cli/bench/serve.md | 359 ++++++++++++ vllm_omni/benchmarks/data_modules/__init__.py | 0 .../random_multi_modal_dataset.py | 152 +++++ vllm_omni/benchmarks/metrics/__init__.py | 0 vllm_omni/benchmarks/metrics/metrics.py | 330 +++++++++++ vllm_omni/benchmarks/patch/__init__.py | 0 vllm_omni/benchmarks/patch/patch.py | 538 ++++++++++++++++++ vllm_omni/benchmarks/serve.py | 9 + vllm_omni/entrypoints/cli/__init__.py | 10 +- .../entrypoints/cli/benchmark/__init__.py | 0 vllm_omni/entrypoints/cli/benchmark/base.py | 23 + vllm_omni/entrypoints/cli/benchmark/main.py | 55 ++ vllm_omni/entrypoints/cli/benchmark/serve.py | 51 ++ vllm_omni/entrypoints/cli/main.py | 2 + 15 files changed, 1546 insertions(+), 1 deletion(-) create mode 100644 docs/cli/bench/serve.md create mode 100644 vllm_omni/benchmarks/data_modules/__init__.py create mode 100644 vllm_omni/benchmarks/data_modules/random_multi_modal_dataset.py create mode 100644 vllm_omni/benchmarks/metrics/__init__.py create mode 100644 vllm_omni/benchmarks/metrics/metrics.py create mode 100644 vllm_omni/benchmarks/patch/__init__.py create mode 100644 vllm_omni/benchmarks/patch/patch.py create mode 100644 vllm_omni/benchmarks/serve.py create mode 100644 vllm_omni/entrypoints/cli/benchmark/__init__.py create mode 100644 vllm_omni/entrypoints/cli/benchmark/base.py create mode 100644 vllm_omni/entrypoints/cli/benchmark/main.py create mode 100644 vllm_omni/entrypoints/cli/benchmark/serve.py diff --git a/docs/cli/README.md b/docs/cli/README.md index 8069110b3..1fcfdb14e 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -22,3 +22,21 @@ If you have custom stage configs file, launch the server with command below ```bash vllm serve Qwen/Qwen2.5-Omni-7B --omni --stage-configs-path /path/to/stage_configs_file ``` + + +## bench + +Run benchmark tests for online serving throughput. +Available Commands: + +```bash +vllm bench serve --omni \ + --model Qwen/Qwen2.5-Omni-7B \ + --host server-host \ + --port server-port \ + --random-input-len 32 \ + --random-output-len 4 \ + --num-prompts 5 +``` + +See [vllm bench serve](./bench/serve.md) for the full reference of all available arguments. diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md new file mode 100644 index 000000000..bcb1df74c --- /dev/null +++ b/docs/cli/bench/serve.md @@ -0,0 +1,359 @@ +# vLLM-Omni Benchmark CLI Guide +The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models. + +## Notes +We currently only support using the "openai-chat-omni" backend. + +## Basic Parameter Description +You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below: +- `--omni` + Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio. + +- `--backend` + Specify the backend adapter as openai-chat-omni, using OpenAI Chat compatible API behavior as the protocol. Currently only openai-chat-omni is supported. + +- `--model` + The model identifier to load, filled according to the models supported by vLLM-Omni. + +- `--endpoint` + The API endpoint exposed externally, to which clients send their requests. + +- `--dataset-name` + The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio). + +- `--num-prompts` + The total number of requests to send, an integer. + +- `--max-concurrency` + "Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up." + +- `--request-rate` + "Number of requests per second. If this is inf, " + "then all the requests are sent at time 0. " + "Otherwise, we use Poisson process or gamma distribution " + "to synthesize the request arrival times." + +- `--ignore-eos` + "Set ignore_eos flag when sending the benchmark request." + +- `--metric-percentiles` + Comma-separated list of percentiles for selected metrics. " + "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". " + "Default value is \"99\"." + "Use \"--percentile-metrics\" to select metrics. + +- `--percentile-metrics` + "Comma-separated list of selected metrics to report percentiles." + "This argument specifies the metrics to report percentiles." + 'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf". ' + +- `--save-result` +Specify to save benchmark results to a json file + +- `--save-detailed` +"When saving the results, whether to include per request " + "information such as response, error, ttfs, tpots, etc." + +- `--result-dir` + "Specify directory to save benchmark json results." + "If not specified, results are saved in the current directory." + +- `--result-filename` +"Specify the filename to save benchmark json results." + "If not specified, results will be saved in " + "{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json" + +- `--random-prefix-len` + Number of fixed prefix tokens before the random context in a request. + The total input length is the sum of random-prefix-len and a random + context length sampled from [input_len * (1 - range_ratio), + input_len * (1 + range_ratio)].Only the random and random-mm modes + support this parameter. + +- `--random-input-len` + Number of input tokens per request.Only the random and random-mm modes support this parameter. + +- `--random-output-len` + Number of output tokens per request.Only the random and random-mm modes support this parameter. + +- `--random-range-ratio` + Range ratio for sampling input/output length, + used only for random sampling. Must be in the range [0, 1) to define + a symmetric sampling range + [length * (1 - range_ratio), length * (1 + range_ratio)]. + Only the random and random-mm modes support this parameter. + +- `--random-mm-base-items-per-request` + Base number of multimodal items per request for random-mm. + Actual per-request count is sampled around this base using + --random-mm-num-mm-items-range-ratio. + Only the random-mm mode supports this parameter. + +- `--random-mm-limit-mm-per-prompt` + Per-modality hard caps for items attached per request, e.g. + '{"image": 3, "video": 1, "audio": 1}'. The sampled per-request item + count is clamped to the sum of these limits. When a modality + reaches its cap, its buckets are excluded and probabilities are + renormalized. + Only the random-mm mode supports this parameter. + +- `--random-mm-num-mm-items-range-ratio` + Range ratio r in [0, 1] for sampling items per request. + We sample uniformly from the closed integer range + [floor(n*(1-r)), ceil(n*(1+r))] + where n is the base items per request. + r=0 keeps it fixed; r=1 allows 0 items. The maximum is clamped + to the sum of per-modality limits from + --random-mm-limit-mm-per-prompt. + An error is raised if the computed min exceeds the max. + Only the random-mm mode supports this parameter. + +- `--random-mm-bucket-config` + The bucket config is a dictionary mapping a multimodal item + sampling configuration to a probability. + Currently allows for 3 modalities: audio, images and videos. + A bucket key is a tuple of (height, width, num_frames) + The value is the probability of sampling that specific item. + Example: + --random-mm-bucket-config + "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10}" + First item: images with resolution 256x256 w.p. 0.5 + Second item: videos with resolution 720x1280 and 16 frames + Third item: audios with 1s duration and 5 channels w.p. 0.1 + OBS.: If the probabilities do not sum to 1, they are normalized. + Only the random-mm mode supports this parameter. + +## Usage Examples + +### Online Benchmark +
+Show more + +First start serving your model: + +```bash +vllm serve Qwen/Qwen2.5-Omni-7B --omni +``` + +Then run the benchmarking for sharegpt: + +```bash +# download dataset +# wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json +vllm bench serve \ + --omni \ + --port 43845 \ + --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \ + --endpoint /v1/chat/completions \ + --backend openai-chat-omni \ + --num-prompts 2 \ + --dataset-name sharegpt \ + --dataset-path ShareGPT_V3_unfiltered_cleaned_split.json \ + --percentile-metrics ttft,tpot,itl,e2el +``` +If successful, you will see the following output: +```text +============ Serving Benchmark Result ============ +Successful requests: 2 +Failed requests: 0 +Benchmark duration (s): 81.63 +Request throughput (req/s): 0.02 +Peak concurrent requests: 2.00 +----------------End-to-end Latency---------------- +Mean E2EL (ms): 56966.13 +Median E2EL (ms): 56966.13 +P99 E2EL (ms): 81016.80 +================== Text Result =================== +Total input tokens: 36 +Total generated tokens: 5926 +Output token throughput (tok/s): 72.60 +Peak output token throughput (tok/s): 103.00 +Peak concurrent requests: 2.00 +Total Token throughput (tok/s): 73.04 +---------------Time to First Token---------------- +Mean TTFT (ms): 124.76 +Median TTFT (ms): 124.76 +P99 TTFT (ms): 156.10 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 481.30 +Median TPOT (ms): 481.30 +P99 TPOT (ms): 947.55 +---------------Inter-token Latency---------------- +Mean ITL (ms): 25.11 +Median ITL (ms): 0.33 +P99 ITL (ms): 25.17 +================== Audio Result ================== +Total audio duration generated(s): 3.95 +Total audio frames generated: 94890 +Audio throughput(audio duration/s): 0.05 +================================================== +``` + +Or run the benchmarking for random: + +```bash +vllm bench serve \ + --omni \ + --port 43845 \ + --endpoint /v1/chat/completions \ + --backend openai-chat-omni \ + --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \ + --dataset-name random \ + --num-prompts 2 \ + --random-prefix-len 5 \ + --random-input-len 10 \ + --random-output-len 100 \ + --percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf \ + --ignore-eos +``` + +If successful, you will see the following output: + +```text +============ Serving Benchmark Result ============ +Successful requests: 2 +Failed requests: 0 +Benchmark duration (s): 24.35 +Request throughput (req/s): 0.08 +Peak concurrent requests: 2.00 +----------------End-to-end Latency---------------- +Mean E2EL (ms): 22576.23 +Median E2EL (ms): 22576.23 +P99 E2EL (ms): 24205.72 +================== Text Result =================== +Total input tokens: 30 +Total generated tokens: 8973 +Output token throughput (tok/s): 368.52 +Peak output token throughput (tok/s): 81.00 +Peak concurrent requests: 2.00 +Total Token throughput (tok/s): 369.76 +---------------Time to First Token---------------- +Mean TTFT (ms): 125.16 +Median TTFT (ms): 125.16 +P99 TTFT (ms): 155.88 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 5.01 +Median TPOT (ms): 5.01 +P99 TPOT (ms): 5.42 +---------------Inter-token Latency---------------- +Mean ITL (ms): 34.15 +Median ITL (ms): 0.01 +P99 ITL (ms): 376.19 +================== Audio Result ================== +Total audio duration generated(s): 3.95 +Total audio frames generated: 94890 +Audio throughput(audio duration/s): 0.16 +---------------Time to First Packet--------------- +Mean AUDIO_TTFP (ms): 11756.89 +Median AUDIO_TTFP (ms): 11756.89 +P99 AUDIO_TTFP (ms): 20854.25 +-----------------Real Time Factor----------------- +Mean AUDIO_RTF: 3.75 +Median AUDIO_RTF: 3.75 +P99 AUDIO_RTF: 7.39 +================================================== +``` +Notes: +We use (audio generation time - first packet latency) / audio duration to calculate RTF. + +
+ +### Multi-Modal Benchmark + +
+Show more + +Benchmark the performance of multi-modal requests in vLLM-Omni. + +Generate synthetic image、video、audio inputs alongside random text prompts to stress-test vision models without external datasets. + +Notes: + +- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`. + +Start the server (example): + +```bash +vllm serve Qwen/Qwen2.5-Omni-7B --omni +``` + +It is recommended to use the flag `--ignore-eos` to simulate real responses. You can set the size of the output via the arg `random-output-len`. + +Then run the benchmarking script: +```bash +vllm bench serve \ + --omni \ + --dataset-name random-mm \ + --port 40849 \ + --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \ + --endpoint /v1/chat/completions \ + --backend openai-chat-omni \ + --request-rate 1 \ + --num-prompts 1 \ + --random-input-len 10 \ + --random-range-ratio 0.0 \ + --random-mm-base-items-per-request 2 \ + --random-mm-num-mm-items-range-ratio 0 \ + --random-mm-limit-mm-per-prompt '{"image":1,"video":1, "audio": 1}' \ + --random-mm-bucket-config '{"(32, 32, 1)": 0.5, "(0, 1, 1)": 0.1, "(32, 32, 2)":0.4}' \ + --ignore-eos \ + --percentile-metrics ttft,tpot,itl \ + --random-output-len 2 \ + --extra_body '{"modalities": ["text"]}' +``` + +If successful, you will see the following output: + +```text +============ Serving Benchmark Result ============ +Successful requests: 1 +Failed requests: 0 +Request rate configured (RPS): 1.00 +Benchmark duration (s): 1.21 +Request throughput (req/s): 0.83 +Peak concurrent requests: 1.00 +================== Text Result =================== +Total input tokens: 10 +Total generated tokens: 3 +Output token throughput (tok/s): 2.49 +Peak output token throughput (tok/s): 3.00 +Peak concurrent requests: 1.00 +Total Token throughput (tok/s): 10.77 +---------------Time to First Token---------------- +Mean TTFT (ms): 179.74 +Median TTFT (ms): 179.74 +P99 TTFT (ms): 179.74 +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): 12.76 +Median TPOT (ms): 12.76 +P99 TPOT (ms): 12.76 +---------------Inter-token Latency---------------- +Mean ITL (ms): 12.76 +Median ITL (ms): 12.76 +P99 ITL (ms): 25.24 +================== Audio Result ================== +Total audio duration generated(s): 0.00 +Total audio frames generated: 0 +Audio throughput(audio duration/s): 0.00 +================================================== +``` + +Behavioral notes: + +- If the requested base item count cannot be satisfied under the provided per-prompt limits, the tool raises an error rather than silently clamping. + +How sampling works: + +- Determine per-request item count k by sampling uniformly from the integer range defined by `--random-mm-base-items-per-request` and `--random-mm-num-mm-items-range-ratio`, then clamp k to at most the sum of per-modality limits. +- For each of the k items, sample a bucket (H, W, T) according to the normalized probabilities in `--random-mm-bucket-config`, while tracking how many items of each modality have been added. +- If a modality (e.g., image) reaches its limit from `--random-mm-limit-mm-per-prompt`, all buckets of that modality are excluded and the remaining bucket probabilities are renormalized before continuing. +This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`. +- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`. +
diff --git a/vllm_omni/benchmarks/data_modules/__init__.py b/vllm_omni/benchmarks/data_modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/benchmarks/data_modules/random_multi_modal_dataset.py b/vllm_omni/benchmarks/data_modules/random_multi_modal_dataset.py new file mode 100644 index 000000000..14ba86cc4 --- /dev/null +++ b/vllm_omni/benchmarks/data_modules/random_multi_modal_dataset.py @@ -0,0 +1,152 @@ +import base64 +import io +import logging +from collections.abc import Mapping +from typing import Any + +import numpy as np +import soundfile as sf +import torch +from vllm.benchmarks.datasets import RandomMultiModalDataset, process_image, process_video + +logger = logging.getLogger(__name__) + + +def process_audio(audio: Any) -> Mapping[str, Any]: + """ + Process a single audio input and return a multimedia content dictionary. + + Supports the following input types: + + 1. Dictionary with raw audio bytes: - Expects a dict with a 'bytes' key + containing raw audio data. + + 2. String input: - Treats the string as a URL or local file path. - + Prepends "file://" if the string doesn't start with "http://" or + "file://". - Returns a dictionary with the audio URL. + + Raises: + ValueError: If the input is not a supported type. + """ + if isinstance(audio, dict) and "bytes" in audio: + audio_bytes = audio["bytes"] + audio_base64 = base64.b64encode(audio_bytes).decode("utf-8") + return { + "type": "audio_url", + "audio_url": {"url": f"data:audio/mpeg;base64,{audio_base64}"}, + } + if isinstance(audio, str): + audio_url = audio if audio.startswith(("http://", "https://", "file://")) else f"file://{audio}" + return {"type": "audio_url", "audio_url": {"url": audio_url}} + + raise ValueError( + f"Invalid audio input {audio}. Must be a string of local path/remote url, " + f"or a dictionary with raw audio bytes in the form of `{{'bytes': raw_audio_bytes}}`." + ) + + +# ----------------------------------------------------------------------------- +# MultiModalDataset Implementation +# ----------------------------------------------------------------------------- +class OmniRandomMultiModalDataset(RandomMultiModalDataset): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def generate_synthetic_audio( + self, + duration: int, # seconds + num_channels: int, # 1:Mono，2:Stereo 5:5.1 surround sound + ) -> dict[str, Any]: + """Generate synthetic audio with random values. + Default use 48000Hz. + """ + sample_rate = 48000 + num_samples = int(sample_rate * duration) + audio_data = self._rng.uniform(-0.5, 0.5, (num_samples, num_channels)) + audio_data = np.clip(audio_data, -1.0, 1.0) + audio_tensor = torch.FloatTensor(audio_data.T) + audio_np = audio_tensor.numpy() + + buffer = io.BytesIO() + + sf.write(buffer, audio_np.T, sample_rate, format="wav") + + buffer.seek(0) + audio_bytes = buffer.read() + buffer.close() + return { + "bytes": audio_bytes, + } + + def generate_mm_item( + self, + mm_item_config: tuple[int, int, int], + ) -> Mapping[str, Any]: + """ + Create synthetic images and videos and + apply process_image/process_video respectively. + This follows the OpenAI API chat completions + https://github.com/openai/openai-python + """ + + if self.map_config_to_modality(mm_item_config) == "image": + return process_image(self.generate_synthetic_image(mm_item_config[1], mm_item_config[0])) + elif self.map_config_to_modality(mm_item_config) == "video": + return process_video(self.generate_synthetic_video(mm_item_config[1], mm_item_config[0], mm_item_config[2])) + elif self.map_config_to_modality(mm_item_config) == "audio": + return process_audio(self.generate_synthetic_audio(mm_item_config[1], mm_item_config[2])) + else: + raise ValueError(f"Invalid multimodal item configuration: {mm_item_config}") + + def generate_synthetic_video(self, width: int, height: int, num_frames: int) -> Any: + """Generate synthetic video with random values.""" + import imageio + + video_data = self._rng.integers( + 0, + 256, + (num_frames, height, width, 3), + dtype=np.uint8, + ) + buffer = io.BytesIO() + writer_kwargs = { + "format": "mp4", + "fps": 30, + "codec": "libx264", + "quality": 7, + "pixelformat": "yuv420p", + "macro_block_size": 16, + "ffmpeg_params": [ + "-preset", + "medium", + "-crf", + "23", + "-movflags", + "+faststart", + "-pix_fmt", + "yuv420p", + "-vf", + f"scale={width}:{height}", + ], + } + + with imageio.get_writer(buffer, **writer_kwargs) as writer: + for frame_idx in range(num_frames): + writer.append_data(video_data[frame_idx]) + buffer.seek(0) + video_bytes = buffer.read() + + return { + "bytes": video_bytes, + } + + def map_config_to_modality(self, config: tuple[int, int, int]) -> str: + """Map the configuration to the modality.""" + if config[0] == 0: + return "audio" + elif config[-1] == 1: + return "image" + elif config[-1] > 1: + return "video" + else: + raise ValueError(f"Invalid multimodal item configuration: {config}") diff --git a/vllm_omni/benchmarks/metrics/__init__.py b/vllm_omni/benchmarks/metrics/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/benchmarks/metrics/metrics.py b/vllm_omni/benchmarks/metrics/metrics.py new file mode 100644 index 000000000..f404a12f8 --- /dev/null +++ b/vllm_omni/benchmarks/metrics/metrics.py @@ -0,0 +1,330 @@ +import warnings +from dataclasses import dataclass + +import numpy as np +from transformers import PreTrainedTokenizerBase +from vllm.benchmarks.datasets import SampleRequest +from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput +from vllm.benchmarks.serve import MILLISECONDS_TO_SECONDS_CONVERSION, TERM_PLOTLIB_AVAILABLE, BenchmarkMetrics, TaskType + + +@dataclass +class MultiModalsBenchmarkMetrics(BenchmarkMetrics): + mean_audio_ttfp_ms: float = 0.0 + median_audio_ttfp_ms: float = 0.0 + std_audio_ttfp_ms: float = 0.0 + percentiles_audio_ttfp_ms: list[tuple[float, float]] = None + total_audio_duration_ms: float = 0.0 + total_audio_frames: int = 0 + audio_throughput: float = 0.0 + mean_audio_rtf: float = 0.0 + median_audio_rtf: float = 0.0 + std_audio_rtf: float = 0.0 + percentiles_audio_rtf: list[tuple[float, float]] = None + + +def print_metrics( + task_type, + selected_percentile_metrics, + max_concurrency, + request_rate, + benchmark_duration, + goodput_config_dict, + metrics: MultiModalsBenchmarkMetrics, +): + print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10}".format("Failed requests:", metrics.failed)) + if max_concurrency is not None: + print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency)) + if request_rate != float("inf"): + print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10.2f}".format("Request throughput (req/s):", metrics.request_throughput)) + if goodput_config_dict: + print("{:<40} {:<10.2f}".format("Request goodput (req/s):", metrics.request_goodput)) + if isinstance(metrics, MultiModalsBenchmarkMetrics): + print("{:<40} {:<10.2f}".format("Peak concurrent requests:", metrics.max_concurrent_requests)) + if task_type != TaskType.GENERATION or "e2el" in selected_percentile_metrics: + process_one_metric("e2el", metrics) + print_text_metrics(task_type, selected_percentile_metrics, metrics) + if task_type == TaskType.GENERATION: + print_audio_metrics(selected_percentile_metrics, metrics) + print("=" * 50) + + +def print_text_metrics(task_type, selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics): + print("{s:{c}^{n}}".format(s=" Text Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + if isinstance(metrics, MultiModalsBenchmarkMetrics): + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):", metrics.output_throughput)) + print("{:<40} {:<10.2f}".format("Peak output token throughput (tok/s):", metrics.max_output_tokens_per_s)) + print("{:<40} {:<10.2f}".format("Peak concurrent requests:", metrics.max_concurrent_requests)) + print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):", metrics.total_token_throughput)) + + if task_type == TaskType.GENERATION: + for metric in selected_percentile_metrics: + if metric == "e2el": + continue + if not metric.startswith("audio"): + process_one_metric(metric, metrics) + + +def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchmarkMetrics): + print("{s:{c}^{n}}".format(s=" Audio Result ", n=50, c="=")) + print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_ms)) + print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames)) + print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput)) + for metric in selected_percentile_metrics: + if metric.startswith("audio"): + process_one_metric(metric, metrics) + + +def process_one_metric( + metric_attribute_name: str, + metrics: MultiModalsBenchmarkMetrics, +): + metric_header_map = { + "ttft": "Time to First Token", + "tpot": "Time per Output Token (excl. 1st token)", + "itl": "Inter-token Latency", + "e2el": "End-to-end Latency", + "audio_ttfp": "Time to First Packet", + "audio_rtf": "Real Time Factor", + } + + header = metric_header_map.get(metric_attribute_name, metric_attribute_name) + print("{s:{c}^{n}}".format(s=header, n=50, c="-")) + + is_audio_rtf = metric_attribute_name == "audio_rtf" + + suffix = "" if is_audio_rtf else "_ms" + unit_suffix = "" if is_audio_rtf else " (ms)" + + mean_attr_name = f"mean_{metric_attribute_name}{suffix}" + mean_value = getattr(metrics, mean_attr_name, 0.0) + print(f"{f'Mean {metric_attribute_name.upper()}{unit_suffix}:':<40} {mean_value:<10.2f}") + + median_attr_name = f"median_{metric_attribute_name}{suffix}" + median_value = getattr(metrics, median_attr_name, 0.0) + print(f"{f'Median {metric_attribute_name.upper()}{unit_suffix}:':<40} {median_value:<10.2f}") + + percentiles_attr_name = f"percentiles_{metric_attribute_name}{suffix}" + percentiles = getattr(metrics, percentiles_attr_name, []) + + for percentile, value in percentiles: + p_str = str(int(percentile)) if percentile.is_integer() else str(percentile) + label = f"P{p_str} {metric_attribute_name.upper()}{unit_suffix}:" + print(f"{label:<40} {value:<10.2f}") + + +def calculate_metrics( + input_requests: list[SampleRequest], + outputs: list[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + selected_percentiles: list[float], + goodput_config_dict: dict[str, float], + task_type, + selected_percentile_metrics, + max_concurrency, + request_rate, + benchmark_duration, +) -> tuple[BenchmarkMetrics, list[int]]: + """Calculate the metrics for the benchmark. + + Args: + input_requests: The input requests. + outputs: The outputs of the requests. + dur_s: The duration of the benchmark. + tokenizer: The tokenizer to use. + selected_percentiles: The percentiles to select. + goodput_config_dict: The goodput configuration. + + Returns: + A tuple of the benchmark metrics and the actual output lengths. + """ + actual_output_lens: list[int] = [] + total_input = 0 + completed = 0 + good_completed = 0 + itls: list[float] = [] + tpots: list[float] = [] + all_tpots: list[float] = [] + ttfts: list[float] = [] + e2els: list[float] = [] + audio_ttfps: list[float] = [] + audio_rtfs: list[float] = [] + audio_duration: list[float] = [] + audio_frames: list[int] = [] + for i in range(len(outputs)): + if outputs[i].success: + output_len = outputs[i].output_tokens + + if not output_len: + # We use the tokenizer to count the number of output tokens + # for some serving backends instead of looking at + # len(outputs[i].itl) since multiple output tokens may be + # bundled together + # Note : this may inflate the output token count slightly + output_len = len(tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids) + actual_output_lens.append(output_len) + total_input += input_requests[i].prompt_len + tpot = 0 + if output_len > 1: + latency_minus_ttft = outputs[i].latency - outputs[i].ttft + tpot = latency_minus_ttft / (output_len - 1) + tpots.append(tpot) + # Note: if output_len <= 1, we regard tpot as 0 for goodput + all_tpots.append(tpot) + itls += outputs[i].itl + ttfts.append(outputs[i].ttft) + audio_ttfps.append(getattr(outputs[i], "audio_ttfp", 0.0)) + audio_rtfs.append(getattr(outputs[i], "audio_rtf", 0.0)) + audio_duration.append(getattr(outputs[i], "audio_duration", 0.0)) + audio_frames.append(getattr(outputs[i], "audio_frames", 0.0)) + e2els.append(outputs[i].latency) + completed += 1 + else: + actual_output_lens.append(0) + + if goodput_config_dict: + valid_metrics = [] + slo_values = [] + + if "ttft" in goodput_config_dict: + valid_metrics.append(ttfts) + slo_values.append(goodput_config_dict["ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION) + if "audio_ttft" in goodput_config_dict: + valid_metrics.append(audio_ttfps) + slo_values.append(goodput_config_dict["audio_ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION) + if "tpot" in goodput_config_dict: + valid_metrics.append(all_tpots) + slo_values.append(goodput_config_dict["tpot"] / MILLISECONDS_TO_SECONDS_CONVERSION) + if "e2el" in goodput_config_dict: + valid_metrics.append(e2els) + slo_values.append(goodput_config_dict["e2el"] / MILLISECONDS_TO_SECONDS_CONVERSION) + + for req_metric in zip(*valid_metrics): + is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)]) + if is_good_req: + good_completed += 1 + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration on the benchmark arguments.", + stacklevel=2, + ) + + # Calculate max output tokens per second metric + max_output_tokens_per_s = 0.0 + max_concurrent_requests = 0 + + # Find the time range across all successful requests + successful_outputs = [output for output in outputs if output.success] + failed_outputs = [output for output in outputs if not output.success] + if successful_outputs: + min_start_time = min(output.start_time for output in successful_outputs) + max_end_time = max(output.start_time + output.latency for output in successful_outputs) + + # Create second buckets (ceiling to ensure we capture all time) + duration_seconds = int(np.ceil(max_end_time - min_start_time)) + 1 + tokens_per_second = np.zeros(duration_seconds) + concurrent_requests_per_second = np.zeros(duration_seconds) + + for i, output in enumerate(successful_outputs): + # Calculate token generation timestamp using + # start_time, ttft, and itl + token_times = [output.start_time + output.ttft] + current_time = token_times[0] + for itl_value in output.itl: + current_time += itl_value + token_times.append(current_time) + + # Add tokens to second buckets + for token_time in token_times: + second_bucket = int(token_time - min_start_time) + if 0 <= second_bucket < duration_seconds: + tokens_per_second[second_bucket] += 1 + + # Track concurrent requests for each second this request was active + request_start_second = int(output.start_time - min_start_time) + request_end_second = int((output.start_time + output.latency) - min_start_time) + + for second in range(request_start_second, request_end_second + 1): + concurrent_requests_per_second[second] += 1 + + # Find the maximum tokens per second and corresponding + # concurrent requests + if len(tokens_per_second) > 0: + max_output_tokens_per_s = float(np.max(tokens_per_second)) + max_concurrent_requests = int(np.max(concurrent_requests_per_second)) + + if TERM_PLOTLIB_AVAILABLE: + import termplotlib as tpl + + fig = tpl.figure() + fig.plot( + np.arange(len(tokens_per_second)), + tokens_per_second, + title="Output tokens per second", + ) + fig.plot( + np.arange(len(concurrent_requests_per_second)), + concurrent_requests_per_second, + title="Concurrent requests per second", + ) + fig.show() + else: + print("tip: install termplotlib and gnuplot to plot the metrics") + + metrics = MultiModalsBenchmarkMetrics( + completed=completed, + failed=len(failed_outputs), + total_input=total_input, + total_output=sum(actual_output_lens), + request_throughput=completed / dur_s, + request_goodput=good_completed / dur_s, + output_throughput=sum(actual_output_lens) / dur_s, + total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) * 1000, # ttfts is empty if streaming is not supported by the endpoint + std_ttft_ms=np.std(ttfts or 0) * 1000, + median_ttft_ms=np.median(ttfts or 0) * 1000, + percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles], + mean_audio_ttfp_ms=np.mean(audio_ttfps or 0) * 1000, + std_audio_ttfp_ms=np.std(audio_ttfps or 0) * 1000, + median_audio_ttfp_ms=np.median(audio_ttfps or 0) * 1000, + percentiles_audio_ttfp_ms=[(p, np.percentile(audio_ttfps or 0, p) * 1000) for p in selected_percentiles], + total_audio_duration_ms=sum(audio_duration), + total_audio_frames=sum(audio_frames), + audio_throughput=sum(audio_duration) / dur_s, + mean_audio_rtf=np.mean(audio_rtfs or 0), + std_audio_rtf=np.std(audio_rtfs or 0), + median_audio_rtf=np.median(audio_rtfs or 0), + percentiles_audio_rtf=[(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles], + mean_tpot_ms=np.mean(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000) for p in selected_percentiles], + mean_itl_ms=np.mean(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000) for p in selected_percentiles], + mean_e2el_ms=np.mean(e2els or 0) * 1000, + std_e2el_ms=np.std(e2els or 0) * 1000, + median_e2el_ms=np.median(e2els or 0) * 1000, + percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles], + max_output_tokens_per_s=max_output_tokens_per_s, + max_concurrent_requests=max_concurrent_requests, + ) + print_metrics( + task_type, + selected_percentile_metrics, + max_concurrency, + request_rate, + benchmark_duration, + goodput_config_dict, + metrics, + ) + return metrics, actual_output_lens diff --git a/vllm_omni/benchmarks/patch/__init__.py b/vllm_omni/benchmarks/patch/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py new file mode 100644 index 000000000..6591af014 --- /dev/null +++ b/vllm_omni/benchmarks/patch/patch.py @@ -0,0 +1,538 @@ +import asyncio +import base64 +import contextlib +import io +import json +import os +import random +import sys +import time +import traceback +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +import aiohttp +from pydub import AudioSegment +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase +from vllm.benchmarks import datasets +from vllm.benchmarks.datasets import SampleRequest +from vllm.benchmarks.lib.endpoint_request_func import ( + ASYNC_REQUEST_FUNCS, + OPENAI_COMPATIBLE_BACKENDS, + RequestFuncInput, + RequestFuncOutput, + StreamedResponseHandler, + _get_chat_content, + _update_headers_common, + _update_payload_common, + _validate_api_url, +) +from vllm.logger import init_logger + +logger = init_logger(__name__) +from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset + +get_samples_old = datasets.get_samples + + +def get_samples(args, tokenizer): + if args.backend not in ["openai-chat-omni"]: + raise ValueError("benchmark is only supported on 'openai-chat-omni' backend.") + if args.dataset_name == "random-mm": + dataset = OmniRandomMultiModalDataset(random_seed=args.seed, dataset_path=args.dataset_path) + input_requests = dataset.sample( + tokenizer=tokenizer, + num_requests=args.num_prompts, + prefix_len=args.random_prefix_len, + range_ratio=args.random_range_ratio, + input_len=args.random_input_len, + output_len=args.random_output_len, + base_items_per_request=args.random_mm_base_items_per_request, + limit_mm_per_prompt=args.random_mm_limit_mm_per_prompt, + num_mm_items_range_ratio=args.random_mm_num_mm_items_range_ratio, + bucket_config=args.random_mm_bucket_config, + request_id_prefix=args.request_id_prefix, + no_oversample=args.no_oversample, + ) + return input_requests + else: + return get_samples_old(args, tokenizer) + + +datasets.get_samples = get_samples + + +@dataclass +class MixRequestFuncOutput(RequestFuncOutput): + audio_ttfp: float = 0.0 + audio_duration: float = 0.0 + audio_frames: int = 0 + audio_rtf: float = 0.0 + + +async def async_request_openai_chat_omni_completions( + request_func_input: RequestFuncInput, + session: aiohttp.ClientSession, + pbar: tqdm | None = None, + mm_position: Literal["first", "last"] = "last", +) -> MixRequestFuncOutput: + api_url = request_func_input.api_url + _validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions") + + content = _get_chat_content(request_func_input, mm_position=mm_position) + + payload = { + "model": request_func_input.model_name if request_func_input.model_name else request_func_input.model, + "messages": [ + {"role": "user", "content": content}, + ], + "temperature": 0.0, + "max_tokens": request_func_input.output_len, + "stream": True, + "stream_options": { + "include_usage": True, + }, + } + _update_payload_common(payload, request_func_input) + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", + } + _update_headers_common(headers, request_func_input) + + output = MixRequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + generated_text = "" + generated_audio = "" + ttft = 0.0 + st = time.perf_counter() + output.start_time = st + most_recent_timestamp = st + audio_generate_time = 0.0 + audio_first_timestamp = st + try: + async with session.post(url=api_url, json=payload, headers=headers) as response: + if response.status == 200: + handler = StreamedResponseHandler() + async for chunk_bytes in response.content.iter_any(): + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + messages = handler.add_chunk(chunk_bytes) + for message in messages: + # NOTE: SSE comments (often used as pings) start with + # a colon. These are not JSON data payload and should + # be skipped. + if message.startswith(":"): + continue + + chunk = message.removeprefix("data: ") + + if chunk != "[DONE]": + timestamp = time.perf_counter() + data = json.loads(chunk) + if choices := data.get("choices"): + modality = data.get("modality") + content = choices[0]["delta"].get("content") + if modality == "text": + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + else: + output.itl.append(timestamp - most_recent_timestamp) + generated_text += content or "" + elif modality == "audio": + if output.audio_ttfp == 0.0: + audio_first_timestamp = timestamp + output.audio_ttfp = timestamp - st + audio_generate_time = timestamp - audio_first_timestamp + if content != "": + audio_bytes = base64.b64decode(content) + seg = AudioSegment.from_file(io.BytesIO(audio_bytes)) + if seg is not None: + if generated_audio is None: + generated_audio = seg + else: + generated_audio = seg + generated_audio + + elif usage := data.get("usage"): + output.output_tokens = usage.get("completion_tokens") + most_recent_timestamp = timestamp + + output.generated_text = generated_text + if generated_audio is not None: + output.audio_duration = len(generated_audio) / 1000.0 + frame_width = generated_audio.frame_width + if frame_width > 0: + output.audio_frames = len(generated_audio.raw_data) // frame_width + else: + output.audio_frames = 0 + logger.warning("Audio frame width is zero") + audio_duration = output.audio_duration + if audio_duration > 0: + output.audio_rtf = audio_generate_time / output.audio_duration + else: + output.audio_rtf = 0 + logger.warning("Audio duration is zero") + + output.success = True + output.latency = most_recent_timestamp - st + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + logger.error(f"ERROR: send request failed, reason is: {output.error}") + + if pbar: + pbar.update(1) + return output + + +ASYNC_REQUEST_FUNCS["openai-chat-omni"] = async_request_openai_chat_omni_completions +if "openai-chat-omni" not in OPENAI_COMPATIBLE_BACKENDS: + OPENAI_COMPATIBLE_BACKENDS.append("openai-chat-omni") + +# ruff: noqa: E402 +# Prevent import order from causing patch failures +from vllm.benchmarks import serve +from vllm.benchmarks.serve import TaskType, calculate_metrics_for_embeddings, get_request, wait_for_endpoint + +from vllm_omni.benchmarks.metrics.metrics import MultiModalsBenchmarkMetrics, calculate_metrics + +# ruff: noqa: E402 + +benchmark_old = serve.benchmark + + +async def benchmark( + task_type: TaskType, + endpoint_type: str, + api_url: str, + base_url: str, + model_id: str, + model_name: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: list[SampleRequest], + logprobs: int | None, + request_rate: float, + burstiness: float, + disable_tqdm: bool, + num_warmups: int, + profile: bool, + selected_percentile_metrics: list[str], + selected_percentiles: list[float], + ignore_eos: bool, + goodput_config_dict: dict[str, float], + max_concurrency: int | None, + lora_modules: Iterable[str] | None, + extra_headers: dict | None, + extra_body: dict | None, + ramp_up_strategy: Literal["linear", "exponential"] | None = None, + ramp_up_start_rps: int | None = None, + ramp_up_end_rps: int | None = None, + ready_check_timeout_sec: int = 600, +): + try: + request_func = ASYNC_REQUEST_FUNCS[endpoint_type] + except KeyError: + raise ValueError(f"Unknown backend: {endpoint_type}") from None + + # Reuses connections across requests to reduce TLS handshake overhead. + connector = aiohttp.TCPConnector( + limit=max_concurrency or 0, + limit_per_host=max_concurrency or 0, + ttl_dns_cache=300, + use_dns_cache=True, + keepalive_timeout=60, + enable_cleanup_closed=True, + force_close=False, + ssl=("https://" in api_url), + ) + + session = aiohttp.ClientSession( + connector=connector, + trust_env=True, + timeout=aiohttp.ClientTimeout(total=6 * 60 * 60), + ) + + print("Starting initial single prompt test run...") + test_prompt, test_prompt_len, test_output_len, test_mm_content = ( + input_requests[0].prompt, + input_requests[0].prompt_len, + input_requests[0].expected_output_len, + input_requests[0].multi_modal_data, + ) + + assert ( + test_mm_content is None + or isinstance(test_mm_content, dict) + or (isinstance(test_mm_content, list) and all(isinstance(item, dict) for item in test_mm_content)) + ), "multi_modal_data must be a dict or list[dict]" + test_input = RequestFuncInput( + model=model_id, + model_name=model_name, + prompt=test_prompt, + api_url=api_url, + prompt_len=test_prompt_len, + output_len=test_output_len, + logprobs=logprobs, + multi_modal_content=test_mm_content, + ignore_eos=ignore_eos, + extra_headers=extra_headers, + extra_body=extra_body, + ) + + if ready_check_timeout_sec > 0: + test_output = await wait_for_endpoint( + request_func, + test_input, + session, + timeout_seconds=ready_check_timeout_sec, + ) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark " + "arguments are correctly specified. " + f"Error: {test_output.error}" + ) + else: + print("Initial test run completed.") + else: + print("Skipping endpoint ready check.") + + if num_warmups > 0: + print(f"Warming up with {num_warmups} requests...") + warmup_pbar = None if disable_tqdm else tqdm(total=num_warmups) + warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext() + warmup_tasks = [] + + async def warmup_limited_request_func(): + async with warmup_semaphore: + return await request_func(request_func_input=test_input, session=session, pbar=warmup_pbar) + + for _ in range(num_warmups): + request_task = asyncio.create_task(warmup_limited_request_func()) + warmup_tasks.append(request_task) + _ = await asyncio.gather(*warmup_tasks) + + if warmup_pbar is not None: + warmup_pbar.close() + print("Warmup run completed.") + + print("Starting main benchmark run...") + + if lora_modules: + # For each input request, choose a LoRA module at random. + lora_modules = iter([random.choice(lora_modules) for _ in range(len(input_requests))]) + + if profile: + print("Starting profiler...") + profile_input = RequestFuncInput( + model=model_id, + model_name=model_name, + prompt=test_prompt, + api_url=base_url + "/start_profile", + prompt_len=test_prompt_len, + output_len=test_output_len, + logprobs=logprobs, + multi_modal_content=test_mm_content, + ignore_eos=ignore_eos, + extra_headers=extra_headers, + extra_body=extra_body, + ) + profile_output = await request_func(request_func_input=profile_input, session=session) + if profile_output.success: + print("Profiler started") + + distribution = "Poisson process" if burstiness == 1.0 else "Gamma distribution" + + if ramp_up_strategy is not None: + print(f"Traffic ramp-up strategy: {ramp_up_strategy}.") + print( + f"Will increase RPS from {ramp_up_start_rps} to {ramp_up_end_rps} RPS over the duration of the benchmark." + ) + else: + print(f"Traffic request rate: {request_rate}") + + print(f"Burstiness factor: {burstiness} ({distribution})") + print(f"Maximum request concurrency: {max_concurrency}") + + pbar = None if disable_tqdm else tqdm(total=len(input_requests)) + + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext() + + async def limited_request_func(request_func_input, session, pbar): + async with semaphore: + return await request_func(request_func_input=request_func_input, session=session, pbar=pbar) + + benchmark_start_time = time.perf_counter() + tasks: list[asyncio.Task] = [] + + rps_change_events = [] + last_int_rps = -1 + if ramp_up_strategy is not None and ramp_up_start_rps is not None: + last_int_rps = ramp_up_start_rps + rps_change_events.append( + { + "rps": last_int_rps, + "timestamp": datetime.now().isoformat(), + } + ) + + async for request, current_request_rate in get_request( + input_requests, + request_rate, + burstiness, + ramp_up_strategy, + ramp_up_start_rps, + ramp_up_end_rps, + ): + if ramp_up_strategy is not None: + current_int_rps = int(current_request_rate) + if current_int_rps > last_int_rps: + timestamp = datetime.now().isoformat() + for rps_val in range(last_int_rps + 1, current_int_rps + 1): + rps_change_events.append({"rps": rps_val, "timestamp": timestamp}) + last_int_rps = current_int_rps + prompt, prompt_len, output_len, mm_content, request_id = ( + request.prompt, + request.prompt_len, + request.expected_output_len, + request.multi_modal_data, + request.request_id, + ) + req_model_id, req_model_name = model_id, model_name + if lora_modules: + req_lora_module = next(lora_modules) + req_model_id, req_model_name = req_lora_module, req_lora_module + + request_func_input = RequestFuncInput( + model=req_model_id, + model_name=req_model_name, + prompt=prompt, + api_url=api_url, + prompt_len=prompt_len, + output_len=output_len, + logprobs=logprobs, + multi_modal_content=mm_content, + ignore_eos=ignore_eos, + extra_headers=extra_headers, + extra_body=extra_body, + request_id=request_id, + ) + tasks.append( + asyncio.create_task(limited_request_func(request_func_input=request_func_input, session=session, pbar=pbar)) + ) + outputs: list[MixRequestFuncOutput] = await asyncio.gather(*tasks) + + if pbar is not None: + pbar.close() + + benchmark_duration = time.perf_counter() - benchmark_start_time + + if task_type == TaskType.GENERATION: + metrics, actual_output_lens = calculate_metrics( + input_requests=input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + selected_percentiles=selected_percentiles, + goodput_config_dict=goodput_config_dict, + task_type=task_type, + selected_percentile_metrics=selected_percentile_metrics, + max_concurrency=max_concurrency, + request_rate=request_rate, + benchmark_duration=benchmark_duration, + ) + else: + metrics = calculate_metrics_for_embeddings( + outputs=outputs, + dur_s=benchmark_duration, + selected_percentiles=selected_percentiles, + ) + actual_output_lens = 0 + + if isinstance(metrics, MultiModalsBenchmarkMetrics): + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "failed": metrics.failed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "request_throughput": metrics.request_throughput, + "request_goodput": metrics.request_goodput if goodput_config_dict else None, + "output_throughput": metrics.output_throughput, + "total_token_throughput": metrics.total_token_throughput, + "input_lens": [output.prompt_len for output in outputs], + "output_lens": actual_output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + "max_output_tokens_per_s": metrics.max_output_tokens_per_s, + "max_concurrent_requests": metrics.max_concurrent_requests, + } + else: + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "request_throughput": metrics.request_throughput, + "total_token_throughput": metrics.total_token_throughput, + "input_lens": [output.prompt_len for output in outputs], + "errors": [output.error for output in outputs], + } + + if rps_change_events: + result["rps_change_events"] = rps_change_events + + def process_one_metric( + # E.g., "ttft" + metric_attribute_name: str, + ): + # This function prints and adds statistics of the specified + # metric. + if metric_attribute_name not in selected_percentile_metrics: + return + is_audio_rtf = metric_attribute_name == "audio_rtf" + + suffix = "" if is_audio_rtf else "_ms" + for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}{suffix}"): + p_word = str(int(p)) if int(p) == p else str(p) + result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value + + if task_type == TaskType.GENERATION: + for metric in selected_percentile_metrics: + process_one_metric(metric) + else: + process_one_metric("e2el") + + if profile: + print("Stopping profiler...") + profile_input = RequestFuncInput( + model=model_id, + prompt=test_prompt, + api_url=base_url + "/stop_profile", + prompt_len=test_prompt_len, + output_len=test_output_len, + logprobs=logprobs, + ) + profile_output = await request_func(request_func_input=profile_input, session=session) + if profile_output.success: + print("Profiler stopped") + + await session.close() + return result + + +serve.benchmark = benchmark diff --git a/vllm_omni/benchmarks/serve.py b/vllm_omni/benchmarks/serve.py new file mode 100644 index 000000000..fe9460369 --- /dev/null +++ b/vllm_omni/benchmarks/serve.py @@ -0,0 +1,9 @@ +import argparse +import asyncio +from typing import Any + +from vllm.benchmarks.serve import main_async + + +def main(args: argparse.Namespace) -> dict[str, Any]: + return asyncio.run(main_async(args)) diff --git a/vllm_omni/entrypoints/cli/__init__.py b/vllm_omni/entrypoints/cli/__init__.py index b233a71e6..2ffba6130 100644 --- a/vllm_omni/entrypoints/cli/__init__.py +++ b/vllm_omni/entrypoints/cli/__init__.py @@ -1,5 +1,13 @@ """CLI helpers for vLLM-Omni entrypoints.""" +# To ensure patch imports work properly, disable unused import checks +# ruff: noqa: E402, F401 +# isort: off +from vllm_omni.benchmarks.patch import patch +# isort: on + +from vllm_omni.entrypoints.cli.benchmark.serve import OmniBenchmarkServingSubcommand + from .serve import OmniServeCommand -__all__ = ["OmniServeCommand"] +__all__ = ["OmniServeCommand", "OmniBenchmarkServingSubcommand"] diff --git a/vllm_omni/entrypoints/cli/benchmark/__init__.py b/vllm_omni/entrypoints/cli/benchmark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vllm_omni/entrypoints/cli/benchmark/base.py b/vllm_omni/entrypoints/cli/benchmark/base.py new file mode 100644 index 000000000..6a6f97eb1 --- /dev/null +++ b/vllm_omni/entrypoints/cli/benchmark/base.py @@ -0,0 +1,23 @@ +import argparse + +from vllm.entrypoints.cli.types import CLISubcommand + + +class OmniBenchmarkSubcommandBase(CLISubcommand): + """The base class of subcommands for vllm bench.""" + + help: str + + @classmethod + def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: + """Add the CLI arguments to the parser.""" + raise NotImplementedError + + @staticmethod + def cmd(args: argparse.Namespace) -> None: + """Run the benchmark. + + Args: + args: The arguments to the command. + """ + raise NotImplementedError diff --git a/vllm_omni/entrypoints/cli/benchmark/main.py b/vllm_omni/entrypoints/cli/benchmark/main.py new file mode 100644 index 000000000..8880e35c7 --- /dev/null +++ b/vllm_omni/entrypoints/cli/benchmark/main.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import argparse +import typing + +from vllm.entrypoints.cli.types import CLISubcommand +from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG + +from vllm_omni.entrypoints.cli.benchmark.base import OmniBenchmarkSubcommandBase + +if typing.TYPE_CHECKING: + from vllm.utils import FlexibleArgumentParser + + +class OmniBenchmarkSubcommand(CLISubcommand): + """The `bench` subcommand for the vLLM CLI.""" + + name = "bench" + help = "vLLM-omni bench subcommand." + + @staticmethod + def cmd(args: argparse.Namespace) -> None: + args.dispatch_function(args) + + def validate(self, args: argparse.Namespace) -> None: + pass + + def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser: + bench_parser = subparsers.add_parser( + self.name, description=self.help, usage=f"vllm {self.name} [options]" + ) + bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type") + + for cmd_cls in OmniBenchmarkSubcommandBase.__subclasses__(): + cmd_subparser = bench_subparsers.add_parser( + cmd_cls.name, + help=cmd_cls.help, + description=cmd_cls.help, + usage=f"vllm {self.name} {cmd_cls.name} [--omni] [options]", + ) + cmd_subparser.add_argument( + "--omni", + action="store_true", + help="Enable benchmark-Omni mode (always enabled for omni commands)", + ) + cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd) + cmd_cls.add_cli_args(cmd_subparser) + + cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=f"{self.name} {cmd_cls.name}") + + return bench_parser + + +def cmd_init() -> list[CLISubcommand]: + return [OmniBenchmarkSubcommand()] diff --git a/vllm_omni/entrypoints/cli/benchmark/serve.py b/vllm_omni/entrypoints/cli/benchmark/serve.py new file mode 100644 index 000000000..906e8851a --- /dev/null +++ b/vllm_omni/entrypoints/cli/benchmark/serve.py @@ -0,0 +1,51 @@ +import argparse + +from vllm.benchmarks.serve import add_cli_args + +from vllm_omni.benchmarks.serve import main +from vllm_omni.entrypoints.cli.benchmark.base import OmniBenchmarkSubcommandBase + + +class OmniBenchmarkServingSubcommand(OmniBenchmarkSubcommandBase): + """The `serve` subcommand for vllm bench.""" + + name = "serve" + help = "Benchmark the online serving throughput." + + @classmethod + def add_cli_args(cls, parser: argparse.ArgumentParser) -> None: + add_cli_args(parser) + for action in parser._actions: + if action.dest == "percentile_metrics": + action.help = ( + "Comma-separated list of selected metrics to report percentiles." + "This argument specifies the metrics to report percentiles." + 'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf". ' + ) + if action.dest == "random_mm_limit_mm_per_prompt": + action.help = ( + "Per-modality hard caps for items attached per request, e.g. " + '\'{"image": 3, "video": 0, "audio": 1}\'. The sampled per-request item ' + "count is clamped to the sum of these limits. When a modality " + "reaches its cap, its buckets are excluded and probabilities are " + "renormalized." + ) + if action.dest == "random_mm_bucket_config": + action.help = ( + "The bucket config is a dictionary mapping a multimodal item" + "sampling configuration to a probability." + "Currently allows for 3 modalities: audio, images and videos. " + "A bucket key is a tuple of (height, width, num_frames)" + "The value is the probability of sampling that specific item. " + "Example: " + "--random-mm-bucket-config " + "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10} " + "First item: images with resolution 256x256 w.p. 0.5" + "Second item: videos with resolution 720x1280 and 16 frames " + "Third item: audios with 1s duration and 5 channels w.p. 0.1" + "OBS.: If the probabilities do not sum to 1, they are normalized." + ) + + @staticmethod + def cmd(args: argparse.Namespace) -> None: + main(args) diff --git a/vllm_omni/entrypoints/cli/main.py b/vllm_omni/entrypoints/cli/main.py index 6a65d9d6c..629a4641c 100644 --- a/vllm_omni/entrypoints/cli/main.py +++ b/vllm_omni/entrypoints/cli/main.py @@ -18,10 +18,12 @@ def main(): from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG, cli_env_setup from vllm.utils.argparse_utils import FlexibleArgumentParser + import vllm_omni.entrypoints.cli.benchmark.main import vllm_omni.entrypoints.cli.serve CMD_MODULES = [ vllm_omni.entrypoints.cli.serve, + vllm_omni.entrypoints.cli.benchmark.main, ] cli_env_setup() From 790e34e375feef2274b303e6a88c8fe0798687a7 Mon Sep 17 00:00:00 2001 From: NATURE Date: Fri, 30 Jan 2026 13:26:29 +0800 Subject: [PATCH 70/83] [Bugfix] Remove Mooncake/Yuanrong connector import warning (#1091) Signed-off-by: natureofnature --- .../omni_connectors/connectors/mooncake_connector.py | 8 +++++--- .../omni_connectors/connectors/yuanrong_connector.py | 10 +++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py b/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py index 2f969b9bf..e28486481 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/mooncake_connector.py @@ -15,7 +15,6 @@ except ImportError: try: from mooncake import MooncakeDistributedStore, ReplicateConfig except ImportError: - logger.warning("Mooncake not available, MooncakeOmniConnector will not work") MooncakeDistributedStore = None ReplicateConfig = None @@ -24,8 +23,11 @@ class MooncakeConnector(OmniConnectorBase): """Mooncake-based distributed connector for OmniConnector.""" def __init__(self, config: dict[str, Any]): - if MooncakeDistributedStore is None: - raise ImportError("Mooncake not available") + if MooncakeDistributedStore is None or ReplicateConfig is None: + raise ImportError( + "Mooncake components (MooncakeDistributedStore/ReplicateConfig) are not available. " + "Please ensure the 'mooncake' package is installed in your environment." + ) self.config = config self.host = config.get("host", "127.0.0.1") diff --git a/vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py b/vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py index c147d00fd..b2e4c85d5 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/yuanrong_connector.py @@ -11,16 +11,20 @@ logger = get_connector_logger(__name__) try: from datasystem.kv_client import KVClient, SetParam, WriteMode except ImportError: - logger.warning("Datasystem not available, YuanrongConnector will not work") KVClient = None + SetParam = None + WriteMode = None class YuanrongConnector(OmniConnectorBase): """Datasystem-based distributed connector for OmniConnector.""" def __init__(self, config: dict[str, Any]): - if KVClient is None: - raise ImportError("Datasystem not available") + if KVClient is None or SetParam is None or WriteMode is None: + raise ImportError( + "Datasystem components (KVClient/SetParam/WriteMode) are not available. " + "Please ensure the 'datasystem' package is installed in your environment." + ) self.config = config self.client = None From 2c294a750b212c9db544ffb46126348d378b5950 Mon Sep 17 00:00:00 2001 From: Pierre LE GUEN <26087574+PierreLeGuen@users.noreply.github.com> Date: Fri, 30 Jan 2026 00:27:04 -0800 Subject: [PATCH 71/83] fix: UnboundLocalError for role in streaming audio/image responses (#784) Signed-off-by: Pierre Le Guen <26087574+PierreLeGuen@users.noreply.github.com> --- vllm_omni/entrypoints/openai/serving_chat.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vllm_omni/entrypoints/openai/serving_chat.py b/vllm_omni/entrypoints/openai/serving_chat.py index 61b00d803..d8e387b5b 100644 --- a/vllm_omni/entrypoints/openai/serving_chat.py +++ b/vllm_omni/entrypoints/openai/serving_chat.py @@ -667,17 +667,17 @@ class OmniOpenAIServingChat(OpenAIServingChat, AudioMixin): if res.encoder_prompt_token_ids is not None: num_prompt_tokens += len(res.encoder_prompt_token_ids) + # Initialize role before conditional blocks to avoid UnboundLocalError + # when handling audio/image responses + role = self.get_chat_request_role(request) + # We need to do it here, because if there are exceptions in # the result_generator, it needs to be sent as the FIRST # response (by the try...catch). if first_iteration_dict[final_output_type] and final_output_type == "text": num_cached_tokens = res.num_cached_tokens - # Send first response for each request.n (index) with - # the role - role = self.get_chat_request_role(request) - - # NOTE num_choices defaults to 1 so this usually executes - # once per request + # Send first response for each choice with role + # NOTE: num_choices defaults to 1 so this usually executes once per request for i in range(num_choices): choice_data = ChatCompletionResponseStreamChoice( index=i, From be580d5c24beb63d3c72161ad150ee0d1df01b34 Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Fri, 30 Jan 2026 17:06:02 +0800 Subject: [PATCH 72/83] [Misc] update wechat image (#1096) --- docs/assets/WeChat.jpg | Bin 129258 -> 127146 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/assets/WeChat.jpg b/docs/assets/WeChat.jpg index 5a15b524cd21a20d4d267812c9534f66b6c8ae15..ab4cdd5c1d3de10ac6fd752067cdd64571092c68 100644 GIT binary patch literal 127146 zcmeFZc{r5q8#jJyFeOZ652KJR$riE<6_O`Qi=C-tosevUVMI~(C6uxhl2nsD!q{ag zvWK!}A7(6Lj9K2h=lOoW@B2L8_c(s&y+f4_0uIPSUUc3;I&JS@ys01iGU8_Si$_DI*EpI@N`SPjUOL?(h@@#1n|K)E)Db<^IBM)#02nq>{ zNFSGxm6KOfKdqsub>`f8LnGr0CKs<x zqsr{O2A0qpzK4&g+}qE0HswWW(*a4<%S8U0cfWB6NU2RrlYS5F&yoF~4J`8iYGnTz z*#8{Y7{Cc(0T&O#10VqEt615*eSaPb8m(hLGNZQ+(w(oSb(fVt?3U#IC>B#J18~p9 z0{t)XdJK8mEfPF&ea%_JZ*=kY#g3wu<3n;x;F9uAXQR#>_s~3R>RorpwplFg?HA!Z z*w7h;jdC^D2*lMzTvyMxEP#IHC0;?*2)x8x&Go;213%VG9C@UUJGi`WuVpOzXSQFX z66=GY2)}4CA-7Ui>+#(3-nL3gT!E>}8~L`GgM3#`ISB+T&C3$jw0U*iA*1duV*%O! zIzH%x%{((_qnx`;V$T;%9se%jEQ$^O{^Z!}t!|Zg@{0bEE$dDMaX2f6&XF-m5t*bJ zhTt1>vs<_|cE;Sr&DC^hJ z%nS_N5XuN%`{VZ99kae7(S>>&q9Eb4Iuj6B8Z#@&Q+%w^|E~5DThrsW@%j#t`H^O_ zkR?yqmk}Jl54p6i4+c_YZnr#bOBdNuzY>sS9D4i6!!0~#!SKa7!xJNhVuL~Qe6yQ* zus5%9IePrz-{NLrLSycBIngKVws!BB^37Y~-wW{z35lT=L;f`l#VaA{4GH z{;r>PBmvqG+@6;z2sa-8k+(X&k@Ml0l~K4|xKoNU4-k!boP z%f0|HA&-&E?M~lko^-vxDscTfM3gh>1vw8En6JLs(S{t;(m6VN@XJBdz6ZPIdJY4Z zwBmNM8(GYZ%}`n2n1JU_+LfwH_AQo8*T1s%H{Eso7IrpWxZ{;OWVuVO^h`Ie$q{yA zmdA-x1FwOXrFAYdN01z7q~(Y@TMFUr%S|z7Vb?hTI+%h~K@dDJLJx@9rNXlsZ#9`f zgDhm_qC$HEJH{tw#qInbZ;V1=@{Oq%rG6`na*^F7AyH)eCY=!Yz>a+bm_}SlqUmYoGRUvxgsSk5&GxqMZVsG}B(rSt`RlJ9 zVgiR+hJ-%vXZ*5bjL%unx8v$)0GEGW}B#a@6b4+n9g`OSy zOZd8Smy7kmow7TpLM))w-wgQxOQa&>+oYJ1+KnVq z0-*QeT^p?N^|gcEziRO{ST$_L4mp^&YgPm5r`|YxUMc$efS%H>tC+!Ux4u9 zZ^-ng`D!fGBExVi*Z4l@ayI|2ZI>?`||OfyRzJ*!|bIPuS^adSXnV zH+_e}t_vrA$>nAO3tf9m03sft$NRE@zyzXiQ-YX)p7kU}XbanN8l%kwauwEToFM?^ zV(eZuL%4i;gtqGm|Lbkq8SrhzbAP*m7bO6^fn5yvConkDr|Gv$fJ?_xEIIpq#dv|r zn8RtvDJZ89_f*O#Bn#MlvyJ=iv@tU>8~#8PoHYN;Lp^!KrMXX2d)|hK2%I$T3jGOU zt`_Y~UrC}L&xkON+8q*LLIQAD|_C>x~sc->cI5`yby`YiTvtF2J60!o`R z>FiAx)jBsJ>W$u!z&`3s*%$oQ4Sg^-8T4-J$*r2jx1IcCjSG ziDhjNQgCc*e`@$iZM_cl<|Jx7%(kS9!xIGZ7jqU3gFP-J{{s zOrY9_E+1RS1Rm@Xdl}Jaa3y(XnLxrwkdB>k97gloXT@#BBbG7WXERQV@-TD_@?bqY zNJB;|enWRD+qrYRv z^H2-ug3lww-1R{wu(FJ#a}F1^z<1@G4l{udl|_t1yFIs7EQ<~sZpGTJaQ>N2kcMJ| zfseM)gHOr%_i`!`_rsy+Bh9sh2HY!;0bK%K`N*TOYeozrG(NGT5(o+4|AHRra1tT@ zVbzb$zZTo*kb3lT;wokNY@Q4x(Kpff-Q76S=Jc~umNV!k8PPEZ7`;ee{Dq;hkDB#q zQm>D^DeD_Cse~wnVqsRECf1;0*?q4-+m?7XNu6Q>)C0Kg@FD6eeThgFu3+xsJzWf9 zuu}WkcmKU71=&6E$qp(KQghRxVX~09$?r^nVXAZ=9OfnyaPg!c!kr-+H{sYRsgT$4|A_`_vmbMUn;tUg2$jE%P8$4Ttz(+T2U)=u?_UCpA<^o?K{keW*E0ha(z zYekYh!z3skyUG>J1YU~QBvFvLT!=+iByHbECNLW?v3TZd!VG~;sUq%KnCZn@t4ob8 z4*o7KlFR##MIcV~UUB-gCx`9y`G#Fd?`1p+pcAFO zNqTmXqQX#AL9dZ+Z`T~NO7W`u(W5QMQ{3V_0u_4i+#Djax+C+*f3SaB;3MASu>t?L zk367j@Aq01#N7{h_vB>F6*obRg%r;66-nxRc(^RwBDL8J8~Ak8mX}Fl02M=mY-7jC=Z`OkmZX z2}oO8f<~q@Nj)R7fqcjWHZ1DE6F2N}bde!)bz`>bbYX?vV=zz@nbri!PjnykQ`x6y z8N(9!Ts(?3YRRh3o~YW$kQF>OyKjE0_?~f zf+~a?6xqn-qOlR-F_q7}%zDkT?4Jcr3e{v6=>JF@Xle`0Q6u4EF#H}qelrP|752Xu z?GX6o<|c7#`l^70Z(2NO{5BuUH%liZc-zhCB06U`6Ns>e)4c0X_Rk+2zu^6LJ~7ne zn@g=foL6C!Q2Ls1*(WpQ_y8FRK*dCYbN(ZYf}G^U zFS>!&TmxFOKg$GVfN`D>#Hf>D0^hulPtvIY(^#IxhEw0z3-g>>UVHRsJghAiP2?#1 zzzV+!Wfi|~{Z;(vH>}&O;q{CiE426d9;w9d??u#aZ}7e0mt4G%2a}{f#m;>M1Nkl! z*vhe^v*NfY@Is74Tx+H(MyJmH$4yO@@IIGG(OXkx2lzrFE>h#Sad-3fHQc0s#LmH{ z5!7{GCXiPcrtPqhNE3dEo;70hkofaHR&$F9n~fyI1l;)Cdkzftn|ck|6fg}rUPL;~ zsQrmuQd3`=1i=o5SmT%?=Odz$>hY2`cVFE{y1{2|5m;mf}j)`7Ns5n`%O8XUu>L_<}Ej|X%QKv9rla&|Z6Lml4!Jc}ASeod=Um>_~ zC)HswP6GAy>91;Q`ro-`7;Bv*aAe3#q+aq5=%O75C*O8zat~t51QzeVfAd#X^zZEG z_s@TDsqOXcDR>o*mik4B(V~o9e6$Fec{57%p(Rk^bgTgr$QLMGac;tIXe?s4=}<EO|0F$(L2m3jw8NW?HFz%;sxBdOZW0p;^klfWKF#Ha*z}Fo> zN2vUjzKku0(lX6CvGt%g74-K(7rFkGdWdRtz;~sb6v0UCkMF{3DQxYuQTg;_i0wr$ z9ZjW9O*ZzarOe)q7BfQQQjgOg)11ma#@Y7SoiZEtIL?~*UEH>b+?mIjJG#R#-3KBG zh`)P^!IN!uAes%5GO&xP!Qe9FG`yxjh)Nceuo?S=Ag4is_sR3e_a7}kaRueLxt*=*kh2S?~m7uR7L&Cex!P(T3s082K zJ=)OpB(W=81&l-;G5Gb>Nt$OvKE-xwiwM8BMQ1r_xAd+>C-p=7Lg0x}v1k*J7zI2A zfq@S^FqigR;gnjkzG%|w6YSRQ%BiAvVU>IOG#H-p=-udw9d{whCzS8ln7hxx{ax&? zR+rzHt(LO#L@E!6?p|bofApjVN*sPL=R(F!aP7ihLl%3#?<7U(@h>12iSd6i5pu6O^d0G@Gm&r8vd&H`G{m*OAVQ*2xV?6cZrx zf(+Ee3FP~nMYBp&PIM>(RDYO2d&y$bJvi4K#ba8|fA7=aCw%Sm2?sUg^3Vr9`CI#p zfVrKmJlKWb!EqBe3$mTKj+tI6-DggrcVz_|H(8!oI+H7gTxk#=s@RI8*Dc*_aI;D@ zo4$CI4c)7WaH{NHf|q4JI&!s2^arRkTDd=FyeweemLOE#j$y^J6D0SZHDs0ZUt3k7uG5Z0B96m$S-%TETemJ`n`|;#cGSr|Z5vnP~no|II^{+e@GMzL=S(xRV*eZ>pjy$p+%Z2N)dcqIs zn6rP%TMo?F;o2WLbELoN6h0vNP1mJDCK1_B4Z#+Gnv@;Jf9bW52Y0P(`IXQHwPI&g2&ml_OwGWSN6^@{9+m45pJ(li% zIR$jw8=QTmMK$|7CK)f8UJ8HAvg@FI`EugmW0swyRnWntoXwa3?pYFD2sQcuy}tVN zAEy95EAv>)xmRfWb2UTT-RV^ys)0|*CUa8&^s>1p$rkIG%T^FyQi#6Ss5x;o209VG z5uq3HvH|lag80x`XkhXk{^B=+U|X(HG#C+#qf7vA!30QS0F^s-catH4Yr=0qNiaG! zh6yBo`xA>MsSrvGgARVS!yU4CMq#1%H;yrZK6oqQPat!`PbrDG=bY)wyhhya%$oEy z;TgG|qPhIGK53JnFQ#}0DUO&+q*OEN;8g2S2f>b{UC9OzIdD)0I0vAlV*y>VZuZ%g z&fx;#L_L@B@ZxewIt7dkQsqr}GJ82V*#{i2{TUa3$Hspj{C@}WmC2PxgcovGItxjU z23_sj;XY{FwbH4{-6LBFDkBPXUPb5OjwtM$Pal}=QDJnU3KnXxv8OCSAD%l|v6>{) z0po?<415+~VQxT1Q|On1143d5S|BF&wlaZrdnT}?bM4m_F!pb5L6@SOSyyV#-Cy_q zE1_^awdbzMMq9Yo$dX`F(yr7`5c@w|!!zReSiW0-1`~KFEe4|ilEeYf?@W*m|9ws$ zt~^rA=z(uI>L@d|I~ZxEPPR@(d#A9S{=|;G5jgddA`l({&KS#PH6eY*3wfTn6SM07 z?7Nr5;PJ;3LN+Cc%wOq8VkBH1bznGlH5k0Php>G?*FaKNgJ^H4q?gtyyj@|4_4!nw zgmKOT^N-Dv<$^4aHx|gw#p5)WQjj2pZ$2`Er3$L!Xe^mb;87l|IQ$TLrS))l4?|{X zO)1OE*a#IDYBG1^spZXtOkc*agE{7vm`aT}9YZzTiLABPj-9RuR9Y%lgqlAs7*Ti5 zXHT8-%&4~PlxIk{xi88`Zfx91b|?t4V$*=_)LME`-r0 zd6(jBM(xqq%ZavyfNooPn@D0o`Kd-=A8Ju)9Nru_)_j$Rk zOwU3!@k}4I2Tx2}0n;GSAq4THISE&?S{FRSvnM%$P9qTLmAr8-wT(Xo38viGpQGk_ zbt7GPVhs8ju;mVp;6@0p9tgYg+tWYi zRtd(be}ap8)aqsw`k{(CZe||S6m9<_enUfei)>XW=((hN?$E`-tOg_~D9UZxci=Kd6-hak4-Z4IVmfKht^~;BH5o5pv41QRH*`5># zgw*mGXawe*(rW!ncZ=`B0bZv%e0{xyC&rDRTtKe#)ZNkL!hj)Zzt1DkjU9%uM0OI-ExBnDIlj1t3F|OvqincET~MFwJmzKQ&|5_7#%eqX5S%xd?-NS*-CxT z<;^wG&kV=zmvdBEO~_coGkOXJ{@$H&gyi+}UD20YSub1o-#Ew_I}1HlmJf!g00&m0 zkm{azSe^YVPwUgxdSXL%?xOi>R__;mCr+{QtjR)Z=-D*48O`t$?twAZbveUSWch@l zufm15_YZ$df+kk(Bw!TwbkKsWUr5%5_0J3&Dl0x@5)O9<96bL*UoxTb)?6Z6*wBAy z-|rd&oISkHNsYwG1o#71%n6~RimR@Bx^-5YYM!sP7GW}JU+#n$C(texEL)8Xhpgo< zjDJkg>5d7I3Y0h7PsZuf^Xd-FxHA+`nZEATBd4nejy#mEdQfO}R=LyS4q8WK+CM3V zabieaA@qn3%rox98jl^yIzjNqlXw<=c!8CQ&al;nNy55Fx$|Ib&Wp16HaAwUAqzR$ z8_)k#>lA*$#}(Q_erC_;9yT7>J`!gN@gP@S5N&rlyH+E0S08e34YjTS_z`q!?@Uqj>?K#KjgQjJ@*KH&51!eSA)6bIs8OT$9M&I(i=jUm4dc{S za$(DNO?{4DCfe(iR==>aFkrpTP#>TjCcZTSPExuZ8z_FW(s+C-p#n`?F@C;&>qM$DYtDtPsHkS?C0@^kg1GFlE{wj9u^UnwRS)eB<e7t+N<(@hHRUOZamfi`^+VrZF&>-j6`wJrN zZR{*JS|2u5MMZMtz;0VWACIC3T8A}0INGm_K$G~;)+7NjD;C&0 zof{?6&M-9^mgN1Z2A9<0X>Z)81v!BhAKEm_oigX2L>hQNV*-3myYUpK_Mdl1OEeLJ zOpFX5D;6RHK*Wf4f;paPTbKHl&*RwPeelC^wLjh@+PIjTgI;~&@5?!a2TP&2UJ$8s ztv)t*REh~rw0P*``^gvZ)UL^_A9C0Tz1aW;NYL0Pw@$A7+t#nrM2Jcc74fc2fX{s- zP|>Ftb$s65=D4;^Nt4LQsqiIfm{Y3-v@4BL-FoHgrs@0T1j$lYgr_ga4#<<=W!gn( zOn~=@@u$Hx@7pH{1*sPn{A7Bf8ld%tTm zIUTK_#8cZC=|L{eC@(4R9c4Acr|kfU>3x0B(ZPiN$Nd^V!0glRf70uJa_Rpay~gbB zd466ZPF}bc$U%J>4G@jfk1lNB;#O8C$DJ7_!*QSr4N|FZDkJSz0n6W&Nbs;8LVTD= zG>^|e@zZAS2wN|Zt1SbMpq2dcq~?-En1JJ~`Oa-H`vvjO{%_Pb{pSc9X_j$eJr29l z!~_a~aDVWAEYxenXABd!D6n_{CL1q#Yv23|KaA}>LG+>D1N~_0to60O#pgfkz-TC; z%N6IGgRiQ)3`J$8s!I8?5`R{}<)m zUWQjJ&@vIAH`jw2VgboIvk&s`+M2>v+*AiG)dvc)26<32G>oPF!1XId!8c6m;7s6p z36?G(3n{~;FadAaE;q%RQC|c`h17skV{uJ?$oEKrJL zc^S(E1S1FsNoEVv8Bs-@g`+J&#ml^9XT4Q!BwX>h*>vHi*%ZJUAeD0p0|RrKc5vP} zh$9S0FPOuWl+i!p=ftLA)Rh3*OI(^eWXpWYr}Z#Nrg5d@i>sp2_3U>iQ@EW~-joRZ z@tkmUaB73w$V?@Sy$(W_|E)B>!H&8j={#&;nH*s0W4Un6X707j#N}^)E(_OK5ki($ z9JrTdsh`KX)L6JL{;yiS_<_B181KPwF;tbv$$Le14o;mwZ5FEUi_lryYuBzlUh7IV z(cui}6*t3IA`_yx#;jvF7Lp~N^CH%_4ffoyoi-rn`0xjUs$xe3=ioF^=HUyH&waIb zfn(6>SSpX~qtDXZ7bGQd5}UM=kh)EBDEO?()8^DIux2z?&;4ftdPu&w>owfK^tat+ zVS*`XN-anZgHV@SnIRA6ej0wv)Vq90#Wcb|2h!LvTz|6R4!sI2Om@Q}KqEx!<9b0IHT`73)E zTZmvIHYuw*QO8vWF{69BU8Ckv&M=D2fB)ygCz?UGBcZI$to~Y`Df9I~= z!5h0}+6OJc+@QbQ#&M#-H2o}bKJ!(D0N30XX{t_wRCd?>0G5z|POATVdt>^WIo(&EbBz%Z}M(f#mp|IlqC#%6Y6fIQmjbIRpP}mtzWquU~um7>62)Pre6N zfz7#qV$TO-k^yOa?#~W-b2O#~MF0{3TC>m=#PplrQoBAL%x9-`KtVjD7B+r`p+jQ> zAMqqUy<)1?+7$;{~UO^bR^wme(((xO}imoF6{U;&g3H@2uZ_>3a)OoZB6G?3181 z#9?a9WA>5RfBG?j$`ZfE+zmJJ6JY-Q2)m_OGA|G8`H_PuKVD;uM%1|kod(`1uD^)* z(W0L0l912&rmL%`BhcZHL6h*5b1oVei@3SID?b)PzxNA_0(~Q6TmK1V{|ad=n+qLa z@;_D-p<~JZV485c3zh!B^<&nqV{-J~n|s3M&Z*|pPA*V_^{j!KYnWsU?$C9|%fjgo z*H~1Z!!hu}Tj8MWvSCvQKA{jI1$Q_P*1$MQb1>(o9qp$%7cG4`L9KqAap#Y4o1f?1 z*#8hs-X{)~n!4pC+4OtjyugXWabf?Ox9SEp(w^LJ)$#o>mU>AY)1nW@hWFz6+$c}; zCOcn`ws{mUZOEHl{~WEq7n{zS%FP#fK(A&2GK89kjRr9ca!&sK=p>ahed~8uiy@$P zEGju4anZpo>T_XUK;FJGkhHf=<2gNRhk7nw;(Ab#sU(V?n7;a)ldfR(5ubqEFwwaI zMzKyXhCsml{L-Eky02$mOtPhz=cc8T@RNI^ywbvY6!^uh$q0nlwmBUA%Ije38JEWz zu)~*pbYsh85ALK7eNPJCkBAAOG}M=q&x4-X0ouo(#Ug#U|CL*N(b<>M)d$D3U zW6z7gUI+P)-~L;K9r@D1h*w`dTNm`tGZA{h^eV($wmY88kE9<~#JC_`8RPH`vkK%M z2afS5qe10=b{;hp(WCJr=mGjTf{sI;um;QeA4aI_BG^wL%2X~wmdBVTo(ItoEQls;sw@K3P%4=Mk5E1kkW zG?@|nNJRJ;e#2A^zsK1IBAiSfEVkrt#g>kDNua#+dGZBV7ubSRmB9$TkvIqzUl0ha zNAK>*a2X~L0s@W63{DAWK`+A2)lhQjD_FWrY~3d{;PJmCB7UkN5rl0GQ21Hp2dDdG zTe$cC$I48fdtM)ug_L`h==5xQAO6s6?Cc*|*ER?bkDG{!{sJWCIOifYrGlD>K9mbg zz|EX0sDxOb00YF`O1w9MLLy@6{&}jMKzIrT;y_HJ=~WEsD5}51MZdeD37Hd^ z4tBW{%&D4|W1_X7rE>ybQQfjQ@4vY%!*OAvss_6BB{;Kba#wBgA!3^p1VS~@y5}kj zRhhUx)A1l&!qLHQQ81PMo{jtZ$E5B+ap2IfH%ZAL@9;xF&JA z{g+0^G0{j}*bA}!xbE7$0Wv6MrYeAyefuQ3=pf7LZ+{Qg0QGiv*gp3&L&Hz~f&+Ah zzm(2ZM)f2G-4PP^?JpuGb@)IiMNZZIYJ1~Pyw~(>8irUB96!lwEr)V%6R{8R|4R=H zu0?5AE*xY*fA+wDZfZQ+P)+Rb@yM9uAzY?nvrOHctvoEv*%~hIzjAMIDuvLIYB;0x zee~jebl~bCV@1(c7;atRIlUY^SMnV@dK^v{Rz&Lp;hCOJCy8rOQfvvT%FXXM zlV?%y^{&Np7t8{%0e%q4hyw^1cHw<7*kF)L57|UK#BaFez`!8;5KFGabm+qi!&yk{ zF|?rk?$?x%&7Nq1?q&-rqT5v3KGDUHh-3_2RMQpksA8srN){2XN?ccn|PjmWu{^l!m%ypXZ)=WnWvsn z;^R~7VSN!-@w_DQO;;mBA`1B=t{;D~K@i)iMogmLTLmNY#Xcw~({dNc&e9`G+QbQ|LyG#GmNB)3>9YvjH444#&{E(Eexkxj5npbKEO z-=H(eL0(2nzlgN{n)GfHR+YEjckf<>0&;{1O@dPlQOSv78T4*)9HUd}sC_k+|73ao zi8P}FQbz;5S(+rAOBX~jN>2DaS+Env8|>am_}#q&>>X~)<)n$t{J0bVKeT8ylKCbQ zp;?~KFUB%vkb$6~An12#{de8LYSHvSgA|BGoddGJX+Kio+q-SFn?ZL^#gEWEERkL>&`90+h7d_@AZ8S6qGIT(nDgIUFS48sxy(z;+nOP`w#PQGlP}1=kqn?MAhjUPy%aK z^h)(>-Ncdtdcr%98y8^o`)&*qr{ zr3=aKtNy#EuZN3c&JovJ56Z3SXb1KS9l8CX>ys4ntaTrCWPTX6lr$Q}FjC^fj=V)s zk61u)D13!Rguj>5ig^}!dB#3Sxh~^Fa7yYK!tKqalr~4ATL*%b@(CGbJmBMfwEj6< z%JJNHs(@+ zLHO`4kG86$tgg}3NaNC(n$O|vG>zD+DLRU%8lh7^)ZVJ-8erfra3&xO$dQikw1`;W zTknleo!nz>Kbi`4l42|$OEmRONi3$kHfC;<^++onafN@aaBfSzam*J61 z76y~e^X5o?3+Q4;Q?_|iNYZ_MO0cLE_15L;r`{~KJe|u^sdx3KQqe$QJDh~(?x`|7o)tUeKI*Y+$bDM7AT zVVlP)N_*axU5wf*eEJCTk-dQzhoUEA=SmI!!=n5K{(lzrf5xJ8+-5Zz-)YR8e3Mn* z@RI8E_K=}eq>{3aAzFR8g?D z)&%wx%s$4pNP*;~Z&r|TFo_9>;@*NBF%9fLO7+Gr`ho_`)&b@bmSBZ84K($0@kq)^ z1nq9ahnEq0j^D7iR?q=L1;aWPJ60F(6354a|6q*ARUzc5cia1q91&7fv~-SS<$3wK zk%Jv5E_Mh0ODy=m*AMc(fz7d}DL;tKy)&+HZa(UpOFLC&PCpl1w^=@j+1^#zZ#i-O z-Wu!?6KIh|(7BVbQ+Os&1p*79{LQ4@qWF=4cWbv2yU4a#^U|8Uk9i*dSZ2BDJD{oi z0P{c}{vroTxl(}knSL-5l@r=VKBMDT8)9Z0+n@3G#NF_7 zkt%D-P07AUxE&IogX#^}Sk4@)Zp-CsC=5{Sw)Z?d)>20j1uJe3^Yei48ky8fKqxK- z@8Yf)P@=VsW~_730liMIZdAn{pGL)%>bFT(3QgZ5lU!)Oq97VD6>C6HVtc zyEiTKb2oS#(hrXmlwpjLiyHqq(Kedy~*OU)|zm`Zgn2I#O4N!qy1Vi}P zEF(q;Pqf2Q2w}p+_2$#z?8JoVs3H@o594w@sp5{K8Ls3`nZ%)f>t~E_A@A1Wh;FT! zwJtqX)t<@iWvzF*a60azx@B)$bc6F?=Qm$6fgF%qg>i(GRsXmi9*tLcl8RbFj%Hr1 z7ndHXz*l_spDa6=Tb`EAyKXoEb&iT!>B!6DPHIDxRfy!^L&FNuPrlnZTyU`uzTNXm zC0H_$l=7Th63#+@GDTdE^C!4nCGiX%D}CxMSsgpr@{}*~e${59gs5{CRWBAG;Sv$7 zZyA!yfp%|0YWgep9}`$x7tBgEIw0YEr->~|YkjNZNynaSx#o{q(?Ev}fsW_c$*xv< zMS%U1o`grO{f~?Oi~d_5O)<1cqhOZXL7Em?xSaIZ4=z%tv{(b!BM|2>mc(Zlh8y_p zDaYg%sw>M4PiL~O-%9a|zR^CNp(PqHD{37J8oZPP_TL^YhC@@17CWKf96B0d#SEvI}qa~$wG{sP0lb$j~k(kjv{XV96?8wNVkNR=AXRcdDA>S#(Yn| zE&jQz>eWKdtb~@Oh7+c!(|2!1qs*1_k!Q!-!$0C)tDA_6schJc-+pwkEcYzm4E|Jr z_FncC%bY!nkEt^o(>}wf3VEyY?Wh=Q_*}?vjlm?F``d3118Mrh&pCxv11F)FtYLQH^QztA;E^BS&kE$Gx z{lKS+i#DWXlRPeJ(L0Z-v9C$k!pRcdY(ibefPCp?%fS-B*zCakdr-}na}4jiT-oz# zr0w+wK|Q<9o3@b)5f|>E(_59m@ns9+I+)iXP@YYiJ&`o zbhXGGHGheh;w~iov>~pI>vXdQK1`3%G}@$Tt{m9#W4Sx!Il)0Ro@W~;3f2`a)`Fa+ z8AcO?GqjL4mSB%sb1&pOO^+B>^}tyz^N3KlT5rcwNoPM6n&iEEH0p@szr^y(s%*kH zft^BJGS(~M@)o7Upub^2hU{LpXZ^jYS=h98U9x6Oneg>Y7C_zXF<-|mI83(s?6*a) zGz->p=S!uWy8}6ql4Cje%YKrTTVeBJL**dt%6t5(pUazuqB~=czgLF7o2+wh=qFvT zc31`6a1HJd1%@z~y)OD!gQaR1i zY2hNG6_(9M)2`O1ZnlPiZZ(FnG}gIuINkV!!HuP4Ho%?)T`l(VedqY409|JV;M&8fr@N64ruVn@6`9d(&>XfU;@t@SiQ z7}wjM_Sa1g*G3zOH8HM>=ZOGx0ap%{R#43QuXO^{WR9aP@ zc;ULtoDpVCBIG$Ys`+NhOp4T_?^0)lZOu4R`l43!;rVz08pMqRZ{~H+tmd_=?K-Kr zcF23y!i<-V-lQ=A}ob!$RIpU&m1P$;>8b7GJ;ETz9RImV9P<&y`ymdx*( z$c)0ytF&_;e2uW%MUgE>0t|>F9f#sN?Q~U?PIua@9a1$`5UU}iL&a2cuOLr4{cu1! zehCQG{^G0pR4V1GX{k;sxFJ;9(SEHg_XQ40!SCseQ#RZBLG-ZqQknmC)uBU*-6ilE zhj6&d_(sJ3g(6R)(s!RL^`(^Vmt$iNo&1$MAk=w=+g?4t$|Byw0;QqWJ%L@P7P2YN z3Qlk3N&G^I5<>O&}*T5}aV0nhe zLtu>Y;I)U~bVA`O%tzL!X{-ebCA|ab(=RRjtzyovH%K8T>=PTQ!BX7c^{#i)6#Bvs zVxUzNKirphjwr6;Nui!ot5Ka7h8oC5^PM_j3S*0E!2hj6sFnM$fpY*8JRO>F*oOPV z=W_+6Q#xj6R2FD64FPH7iyt?Sbd&r;RX5fKzWT23by1sRfn9Beu>1-mnt~-dVJW|4 zXe!r6=v+il$*%jIR#LmbOEms=37ptzjoi{`0man*8l>V%hTq^N$n#pc<$u$at7j&!{^{ z9x%o}u=BX)df@H(9{sQ1=6Y1HywF*FIN!Bwm5oc zMFSn@Tp-Aj@9j&X*oOmP0R3bP2Bdq)=Z`@@nWQP=0c_t7usSlOpU)5grQ}6#u>FoG zVZ9&?76m~$_nr z^b8Gp^n5*OJ8-9^OZk#Ae}4rfOGr9Eep5iA{7?utJ3Grtu8%a>IKPef9>p*%ibnkW z$OQU5P~8ecSeH3h%CG5{Z!VZu2uS$D-I8xHq{97KcIfP2Sq`m4aGSam)GosPC}4Wr zs6OpdhWY6+Jr|d>>ukmt{sp|&D(V+FhN}%=_2`AMEjR^!NL$7L{II%DObB;I4m?pB zKUfeTsPdk~(NXbSPDpAEk`tv91PAq<7Gul>u$Q_L$6zsJ0cCGi4vx3dABiQC{65G$*7e73$f_74Ww&-RFI+kgly0X}!DM>wq zS_`49&)^9MXT=a5)UTP-JCy^gYAa7qq>Hen7r0Lp-?a&n%$wrmHZuV>d$}+IByv8^ zW(*(Wb34^U9~Jkp$JYod{e*fw%-jH(w@rdZgN+IZF@yKEdL_EAUO3O+Q^$YrKemjP zP>GNFYOoD?J<~9mx9=k|QCt{LifR54biSX}l`$>po?$d5R8V|;f-H3uqH`;LB)}o2 zt(U)hDk&zU)CWyX@Ll}x0+QFz^Fbaw5U>{U*2GXv`0b;zQ;tQeR=rh{9**P|O`NP=-& z^q>~%?b`I>a(m;%3GdX?R)L-3_fIvEaa`v2!x2H(#CeC>%~ZTT^KbbO6iG>~dh8^{ z#0S&3W8bsBd;h9QVJ(bOeXU<`-T8*r^4&8*q)K`|Lz10!IWV4O@pk?K*jZj0ewY%z z-aN@&p(uD+`B}GVSlOL7{-l%*jj@)*_s5hMTq;@R@I7^^wvJ(NA`EDI-$^k zW3?B7D2w|OqYjM)>=~!^&2Jj;?~>kNgh=GksZ)<1y-upVbs@!yrl+{W-cdgYR7j)^4w^{^*RIT& zSrW-9$#?Xqvcu%Mr`lJ@3SCbqSq^@O979d}#0B^~s}J z5U6u2P=XrChKt0C*V(_P84URQ@TgZ3UgiC~Aji@C?eqaoj>HgW=x{?7$N{ng@z8Jy zhlz6>lq*-xDcpxi+4JA^O*J=IM}|{WO*79CqhkstYIVp0r|dp|TR^E8y}8iqt#Gq0 zmz+p>RE*IkK6v-_5$7dr7$P~%baVfmPjOz09r1qp;-!Wduw29iCrL6QGjb_xpeYRG zmglzWo|}=0I(f44^-GZl=^fwaGLGKU=n^{pGd8##S_p(x{@!N7zkz7aO=>vV$2jVl zr4f@g+}>-RJWl&|;++EbTT^bm$Qm(8at4X;zR2rzzOP%TYa%CDu6DgZYg^M`Ne*KI z*+AGALt`(cTpzwX=MeUeS4LoIP4+!%#{3boiNQXszW8o<-d;ASz-;{5ljMAS~>^r49}cF#V0I~xe=zkO)&T1aA@ zReV&}ujAWez`7^5vZw{ei+<+#4J=nIZFrtC^dg%at3Fa-G_n2>w{zjP+o0c|+fwcB zCOM1p35SMV>)*n1oj7z!2ad{EKF`bh>FHTENyyqtt@i61TpW2-E_xuJz} zX{Hn7OWZLcakZ7%o1+t@9tov$rmEeF-;A_K)^-yHh5l&-<*L)~kZfuV3drmYarD#% zaY|hC-Ic7Ohi{SQrWF^CMS4qXn@Go!MQ0ikUq&Ew>kNq29kt%2L9RLC^$izp$yc9F zd3x+|0B6wvboq=ps*s{ejA<_>sp{7as%gJF^R;FGibU+-i>)#ctJGCMK17h*}H z4KOsrga`US{n`or?sE~U%b^@}zQQ~hKq)8oA_oy}y)a6US)xL{_4!kmPADIKdiMNc zldgI1^Pr~n*xfV+zf3EP$`^p$BcfK<#}0Rl;JIlm^rFIZqM19^rY;9NyPA6v$GgD) z*|F~sv>5vQIt^TbzbwD%Ps_J}Rx-d{bUNqq(2zD|R8@2Hf}Oei{^Oed9H}Q4AbHZ1 z^?T33)>2nBOX(_vU4TzyJt1 z<_`}!8A$*5mEE&{s`Egl;ijO@d;QP7uW{9Crt5@DA}tm?Pe7H&Z}--M_b;yaop$$q z1U@UJgn(X(QJv&l)J<50nBK5yiU8pgxmdH;6Bx@IX;x+Q!+E5h{?tADRoYcbqCEbA z@ZtgC#vPm${du?=Ikiqd zinTT}mR-I!n0z(cM0XhcZzSmlFt~uv*&1-sn9$k)k##`Iu`~F&0f4L9aIz`~H8v=x-!A&5p1+f|r zXhn)3tbs$V0!KR%{@6hih%}H(AXY=tgXGb1L}3?)7_{RjJFq5R*#)642)<(RME?I{ zpMm@@&(ZwxC>%G1?j=CI(7QDAN+olfx?V0nt{CxK@MWdNro~V=7PKh*Lk~vbVatCb zjF}+610akUCkNA4HRTw1X81ga>rs$k(SOnd{w3s7A?t;X)P{QAe=;mVs!uan>qK^4 zl3Y|?1|1-ERCWr800v8(fo@2Wt!mrGxzdtSh6{bgdxd75#)VqBMdkX)$A>rNhn{>* z7=8S;uBj&O1v>LK_pJjijD0#-DThV)>7`~6$OTOT$~c68`iy+2fHwBzB);FJgD=){ z60gVN^yewS^2@Ym;ckZh-G4CPEqIViWU31U`=A9H#%W#vPCIaN|Brp#Y)xG%Ku~=Q zEi=7X1XQmkzv9b=xM=$2<3$L(jTo!Ed2TGJmv<+DTdHix*fX5=KK!eiNz5e2;kJ>; z`(~LOt^HrAbcuzJmhIJTZP^~Rms9BoI{i&TB-y9mX(uns^VG76?!O_|*Bq!d{<1QT zYFlffd-&9#SK?FMM=q)U6*qm@MB$ zENS0<-bhUwb~i~tZF!^~5rMoyWVR*VYN$Xp-=q(#VZD9iRo>D@K5u+OI`Zv%&JXKS zi;O;N0rD~hVbYePa?7-BDNEOob4{4x-^ z-Z7??;czjaf`>Qmw~lL&s>Ki#Q2-DK;j==nWO!f;pCOMIn-JZ-8!&-cm444{2rRSp zl3A{1yxw^Hm}o^Cxg~=lXVUlqd|REiuq%@*U1jau=y&LyjGU+lMNq=$kc;*c_Gk5E zOV6VCqo8uG0%X<_bveJh^#kpWra98A3XNq>qM7n&sFPb+!*tHSp^zUV$@@d8!Q~ze zhUnpPUWlnILB)HP?(M1Qug2vvZ*NM2an4gQ=d8KGpGQJ$|wbDND*M+u>r zr$?Rh6q5a%WI8!Oz;1IWZ@z@Ie^?4ao8X>GXiSynN>wXRRcDB(l4^S_#TC+t5jSDxJQkwdlD3jB&Wt$#s)?2{wUpl3uePZQ-! zJ&Y?&JCEIqo+S8DeDe=62=e8?6=G0^Si@0S33O4jOtI+23;l=@sS~@IseNIWjD4 z3#K}43ys=rZ79=^QbZt79Y`XMk#PU*0m zlqE0IW+_B6;TeknH*NJLP~#{-Y90%lO`Hf*TOnsZ3Y%!A3k@Z+4I#p^GWFRljB>$Vo}o2YhbhkNt2 zI|xtj8eP(q82wOJQwQn2e!D&@?v|*i`foY=A4qiRAL50goCom)fN_!8^vY-1=w$di zGk!H5FV5}F8C`WL#v6QO!=!L(0!IGuRQWUE0cNdokpk=M@lx)pROBp_c`Yo;QH2aw zrP#U^A5*ixjfDEsofHypexWC#j!X*`${?-_l_sKf9XtAeoKId)G}n_F!vO|J)?=m$ zx!8QaLNj%Y$P#lPLH{%)V{z{4&GSnl3%##uf3{kgG53?o(@{Z=H_Yl3&-pszU*G4# z&WB1zcib%4HV|-6W^CZ%)S)DFm-)p$n_dgex1T6>D7Q^Eif&Z+RB75;El^82OGZl4 z#`8zjNu>0E=Ex$*7#Jvv`jTgQ-fyO{VgVFmRi@8HC5n13lsaIDEVL;c@Zns6&Bgt( zx$(~>{7*@q(n2U6LX1FHpMntLdGhqp63wSf=u&)w+REcEQFN4Cdz5meKUUB5@R`PvTOyzMAIwoR@udvvC@2bsvcW(Zb>z6jQ{$pW*Uj{q(Q+T6dRIF~Cik}h!xy5RVsSRAB;!ZUEDL(K}EzofK_xN>wdL0_FKt;|^ zJ^7QsHT>}(_6NIdKbz$lhL@9|Fbx)__z>#&t1AqX6Mk7BvZ9zWRd%&PJ5bNBU8iTO zw8iq|(!IafKrySQ9l?3=z$x@Dn|I7ttN1mzFpZ6(=pz-esKw`G3F^eRBvY^IccYv& z6uu*_C_AzK5ZtIP`F!J*MFj@3Wu^^+abv)T2BodzeshAU4`=((Ra*{uOux}!$nWQ` zZfjhAM^76-svG(!Kxw%WAZFrW0%}}AQe3DyV$Veo*E2tU8a>UwT_B8Br;3C0?xddn zb_(^UO*Kdc`a51L?m~m7uVJ7ftu>oydH%~-!9&(!6watxz`<^_{QDv8Q5C-Dz%t_Yjp~qNJ1e!A z@un{cKe#UnSHQ;$AH9pZ!Y*HPWMWR&%yc;tW_A&?r}kd7YNH~E@;BY|CX?P}0Z7SR z$ZbWM|6VQrkAGX*W%3V;r{y}7Y^9ne^^i1|rqwU|a)i<ep4>wAw<0l^qe!G6qQldtpaRL-6lzXpz&a;vxhVP{Zuh zp-@$RGAf~QL4VlP)}ywrwj+sg=3&(}o}#`4fsmY~1-cECi;>+`Vxp}Zhh)3ligV+N zrIm)|8zq7bnPWunh_8Ci7W0Mk@n1IT_Dgncb{rG*Y2rlhB{(OX_v_Rw-SXCts=QNK zJUJ3Ar$vI^Eb1G&pS> z+3-1No>v4`RiUTMuFe<0PiiNa0VqrB-{A-1Hs-in;8KYwE0aGR)vb=%)ou*hoKSqI zNgnvK;?0}(cRk$Tjx>}s6T^JV=&wcj!+t)m506m4T=!Fu2YA$4C5|LlRn&o>L%u1i z{^%ju#q&zGE(zO%&KY^jQVK~BTOFG}Jdv{oh2`$HzaWPrTPMoq*HOPvoyd?>?T&AZ z^FN*@+uKJzqfI{6+55T98bVKgkzz%#R{@>)*8CJ&!%3jf1~IG?AV3jPoj~e-0#M5l zPr^~uXRL){CQtN4E((t z>wtNCIkMy=Pg^D3r)3n) zi`tVq&ZhaGeKuafx@5>et zaiWiM?!&7-35G4lD_$MbV>dP&6}L$+HP@}K2%oW&8{4{Qfr3;*B|f!CVbX5tVO8E6 zrb8qs8vry^M3(b8_}T~2I3*~#$LD|>TjqO|-?|Ba6Go0&ZIeDV*o>0A06z zobAcifw2$OZ5nfQXuBm;!e9a4bw))TmHby?y7(W^oH)C%CChvOV6ADUy>I@JzGgV{ zJc8!P&dnO>f||x^mx*JYsH*c_hi!}RzWU#5<o(9)R354aiHhzRjj{ZC%)VH7@|3 zAO&!xO4lBqu@B2bad1_fo5(vx==G6x6=vFPkG`F#h68EHqH`+(0uNZYKxmH*PkwIw zJ1xyo0lNuVNB(KZXaq!7fQ$TT$T(b4AZ=m6X}2ZXUCjzPEIOPi4xO6b;8kVdqkXah zvy97I&phWXUktWI9c)W!*@n9T>wPZ+N#`z{939KM2ngzP<-x*b9Q-eZ*rEf}a`#KjaOO1kzeP|5eTb)#+dyfO|-n~(&me7^(U=v$U`=yZU-S9gTyyyD&hobZR z!=Yy#+@Lm7ME3b}(Zoq-I8Pdf~K}JVbd!7S!FtDhiic6Q8vw21DU@dR73BPzift)m=_=hVlv(zU}|R6 zbmms3wKH9Dr!L91Owd-O@CHFsxyL`>?%$5E$`8G-#$}>=(|hCLG!o6F+>CT|P<3`2 z((yl##Q)}tGnJE3xUdNes_(0fsp|Q6-*2_32)%(lKd=29!WpczC{w&6>ySNQD_+qr zl0&lW@8NSdW!wH~%$l`Lo;m8DE(Cv8AQ- zROZwnB2EFFCC>*Rt_D4OM+*d(06CE(?uPUQ&yvZ3YjuW;aSsf)X*Gf-H_}OlYJ3Mg z+~js-$ zpQ)R9yyeIW_5^B|(u8{zz?2zT91#1%Tvj8ipO-dNU?A2|g{Y=U#4k^w!z|(dikK^gwq(ES!$s*x3Nz9P_7yx8oxDXP;!Lgz7$C;N7TZ7Pfl% ziqqQtWYFK~6WO2_tV~DsUp1XA$GeLb_he2`v33{Yr`Vjv2Eh<`QK+% z1xhc%bjhP@0yZTD(m576#%~^P?_#gd?|C{LNR=-|_vK;dTT8hpLP*kYQ6RWD?GCi3 zwiZ7|$;x0khB`R-zD7Q(8BI`mAl5B5E7pbGVP)BY+=gX6C+X!P+q(Ks867EPw4~Uy zFCSK~GjBvm)}8Tk!X1G==gAml|2JY+UiJgkX1;I!selQh2)~RIG#W-{IIJLRJg?}R zwyCo*$_XyrfLkNNi^V>BI$P?&D2%t>l>HXtk~)om%$~)RTSqRnx3P@~T{3)V8FiZ) zCm0nI3Y8iv^E&P_PWep~@@&yaSxrIdHAEY!3f&WmDrt}l3V>`wm=5^1e`1_E%^0xJ zUC5<0&#x;HbJ-CK1L~QNCx2M61e39!$Y*Q;VIvGpH*?m-nljb)quX|k{<*Han~&-k z?pcURTd)e&OF#CG3cBSf4IyiL8yUVMVJs6VofourBYrvgy1=j2AWEJ%)iVD)gYTJF z&obn>X16AE?NTl8TE)jf%U2phbQxpRr(>SQvOws7o^yiyRyJvK`%y|`=IA0ke5EE; z&GH7hRqo5YQWKRg4eSw>-6$yFkP!fuNt9RdsUnxXO~L`0%&-6-Bzc zq)(vh&_TM}ikxXD@&zriXW@?49Ac=Q_akR1U&upluD@{R4(NE>Lmq`sUF_b*V`Zd3 z;BWJ<^W}N=5rF2At*6OlUpznaZ|B#)H&T8DQ#Xw6>$=y|skjVr@m^+ z()2Tbd+}xbUKQHA{mP>LkT20Fi`ucU7MF!qzgdt#MOXYfc*!NN^kGRmPbGPfOpvlS zUliyJhz(=F6+R~RjB<}{o4ra9Ec%@jcDHg!;5Q3HI;V4+oc(>YM=L$Mu_yjP3+gTI z>o@r>05J0Td@7h*t$!Y<+xvqdDxIYFfSd&WdaGe($uaBo{=*6EMttJ~!z6uM7_H(j zs_6zf3{dGgl9o8!-I2kBb`Zs}$}vr7d(U0$>)M1F=wrTEdl`yzlcaRp7DdbDTCy(y zP#7N7$J$;Jkj{uYn)QFILxEa={B{&3(&NnRdsi}FeO*&$slEsefXWAun4Y;RT#%I( zdGAZ{`In?ve#g15zq%&>{rmpHH1*oj=;A`etFNhQ z*RQnQ@9H!@H^vaZNd;tY*5OYQTr>eEN*gmae+xrblRXV~+u4V%XC;m?=iPZ|w-n4m zt~#9~32O`*XD{8qC@WD5PlKK zVR2?V`<( zL-@-Z`x0|)DQVlQ{n7PZ!gHk73LbXB-H#xGKmPrWL=xOvK8(pTY--}~&Un%Z;;qNC z&1D9tA87NXkFvCFMOEr_xDTFx1ChUiD1^ThEREfPUz&mDt6_67Dcw*6 zu(J?f97wb84TB^D?*s_8ivB;4G_p#IN?$hR`P9mUo%F87{4}++(u3^^#%xV<>Q{2X>jC+!SjY}?h;D+%S zNz&4k<-6~xW5a6LAM76nW8R18GbB**VX#X{lR7&URO+Dj z5Rt#trC_@ddUdFSo_3=Qy&!DAbJxJxZu??C;w$@?4t^eJTbGL0APEGgEpcej#{TeB zValRgG=H_P*XB6V@w@(ch9l1zO4q&U{_dES%)?a$46se)FOaD8;?2RPs}XMEv`cNW zNDqN)99+&BHpNt z)QX|J%96dGo3bV{w)*$owqEEBISu;L_noNVfDi8~M%%v-N4~l>a4phn(oi%S1b>WT z<|3(hbHKbXMd0@2SamQR&U3Sqp}{-Qi91#j;Ihhw{=M>oc-jc2l?&9~8niCJz*n_!Yb_at8DQSz9iy&tzN2K;{H6JTTA+w0DY z(ulI$@p9(DedrOQEQz^dEo>_PSK-P!ewU$Qz5nIu@PYN#Q7d zA-*eaaC^4>%3{CqIny?BYl^xYagsH2TH#l4900; z)_}L0HIeKnyFRu4VMa9{7S3Q%`K;iB7|AdrlZ(#|M^HA-3a^0sP0wzFN?ps~_*ghK zr|JgVb){|Co4ln{w6e0y9NoE*=+QDgH`W4jASGME*F*PA^}*qgo`!Asm!FcOyaq!WX8-DD%pO@Fq_ z_Af|^{a+Bi9VBkm@7fabeQN&t_VJ6wQkfaA8V6~8H-~(SOvPtW&l9^BxNaXn$z&nE zl36cK494LVJ7d3FL~6bA?_krOF>CZOLbV&XVFqfuv?o{@VlaY`1qfTq zNeAZJ4VZ3I%=wHmQ(Vf+x~8#IUSkQj_|JyNg#NyVt91wDizI_u5Q>JM|7;}qJ8S)) zKa*gP9a%B*cjcufUW`hlh;PZ*7vKCGbj~zKH)1_boewI(49eg!{<5)MM)&OQjx|Q- zp{U4{_doAE-)5GEutlxabB3R~&8J{#I&PWI{BXLBOjS$}rqXxO3$3J(E|TXkG~G0l z9xnTGbEQ6g{`(A%Nje32RyHIRaweMhIXOUfcX=Ta0hG5U{(|U_$X5D%K%cbAu@Y(IGtWaz;|ci1Wmse_@|SmcAOLvEZb$P^`7e?)`lovU1OmIMZv`q2wvt$ z7ZIb}0P|7y?Jc%F_SVmFeo<`v!@(S+AkaU9U%Szk>Bc5z7mr#29fDe#O@S zJ;ZT67(J)BF)?wpAt?JWKVMjE-;tv5wLv6-s7=GDdFk1Ob&VgWc36qUr}s zhoRl3^iQ;>65DRcMYm>Y;(5%jw|}~9+{ic|AVz{4ricJ+>EF0mK*`oyKf{CX;fO+{ z6@qiiUkWo=l*ygqSpIb*{>nN;HUb^PLMPR2gmY|ZJqtg;H-V8eA$vYvJT;9Ckv;Jt zMkLDY_D>@3rjhWw;#z>@Z%0w;smE0bCckXm-?zXZWGZl5 zSjLG{ns8rbxjq&2G*tJF8VSCTn(l((S>acFlS0(#0ZM;Q`NE8<5M6sXbDuJdgAgS6 zW9Mc;b7RQ#2w&>bD=Ib{(f*D_2pCL5_Mq9}zDi}Y8zt%@EMHaenNs(bfxTq=tzIYe zoG^4auX4I0rm!R~fWheBRI3Jo>W6g);v!DekNFX1Zeh}BQ?fn*4H5M%w9!qEa!ozu zZh3fC_ZD9}PLM7tYYypkSP3qm*(*)%dF;a|rpTcQ80mFOB4-yQk;i>-ti(}hE0Spe zH;OL!_+@j8r!J7swzDuV)2GkCoddN5pP) z*K+~Io_{lJk~M{|=20~YWQ03uy2E|E2gg)4RTV=YSZ)hbJc@3%o@h}7<*_*|D>0;W z0l%a`yx#KptbAKUgvd}gS+<&PJl`ohZ?tat!sLp4YrHv)4#hOGnxf+yypza=75C>! za&i=RW~-dN$h5*pMK{sS=H<)gafHZngU5gw3Qk~7NRhW4pX#JIVyrCu$7FzSL0cvGXajO7XcQk!1>QHxf?H(!U5J{BPmh%5JWtIIQm% z@Ha4IKS`DkLMjzO5UgiKvL0VNS=*Ae(Hr{nzlkfYj<I5AEBJn-RhpT8i!ssI2)jljuU`0c$>ZHPRqBMpRBTyLukI^H&gxz$LeopJg0qYB^2qb`PssNPm11wDJ0*Bkjt}YgRv( zSsjX)IbT;?q+EYhEaU70*vokn=j^AwgOHhFgJS;yIJ2xVf}j$54@>viO-ANd-n9aq zt5+UapL-aHV&TGj8}uhW!1Q4sMkQPm=xINvwn@2dKc!$fe#w#Tl|a9Tt`FOI>va*I?elP|A{`or3^2^@aqJEM&T?+2xdOV&n|Y0UvGHQ| zts^Ok?E{EGJN#0MByL02r)+w|yM3cp-{bMLGC5^^hogbB5EX3sddmu8AU5)@-+Ts^ zr=#-El_Iz)c}bwBUMTvS)r)PV^8qrkq=dKFZL^yr#Dy~TBBS2B$~T*TrI{KA%R!{q zW9)xp%~RybD;md|Epm+wjeDKnpqv@l>=P89s}msCk(yxkS(I)Bgg^K4%l{CWmdypc z=-Sdn^H4#b?8Z(%FVAERY3QvGb+)RQl#^RxTd5~JL}sUdUPfV{A0zeD`lxsG_ioE& zqi-3_)%gX*OdV?yLmycp^o0GY`own4FaB*uSsgn*9Yj*Ba#zcSkH=@eG`8(KUt4o) zQQd5pyHqW9%w;z@*;}|x&4=D(UQ3;Vx%{Bj%~oquaC6C9^s}EpTJG@ZIEBp1eRwvo zy3oO0NJ zAHhiv`16W)Lmn$oLrM#M-EiTyKIb|zdFmU~(xpS5-b;Y^Z@o}zL*xK?8Dx0lZU~Xn z^VL@SUVij^?dNBFx(%Oo!thUIU5ueT*@(z8hqcCri1}&BWG_Q*m3Kn@S`e~{mi)7l zuaSp1^F>K}=~_eORnt4OVTPS7B;Ta}{8cOxXS%?E{AKqIOswPiBjmf`1PXzd-3E*o z?|fTwc&g-D{Hw+H;flo$hlx6ubvJ~7liG^F#k5r7uH|e#_WzM7UUx;3!~ZjYkgvBl zy4}GR3CINXwSu`vo$EL>_R2F$=Ov%|_DYAMFw*sk6K?|q5T9lu=q+ipYvg!F>N-4} z_VGwDdrsET8`~5hC5=e7REeHqe^`}%QH4_Cw|V7G@Yn*A{|r-eti6-1z>izzNi?+l zd}E*?{ZFTs?>=6ippK<@M;BRub6npR@kEG!vCqQ3#1rbVC1F5xNDoX6lLHxeu5uQ#AZI8Mw+rFsvvzzsTB+0NhufA7PMGQ2t ziwds&{ehu`0fVgL4iI7as3c`??)i5a95HMhs~oA|AHDIi_E8Z|xYz2&V{;Q{&MUY$7Pb$Rc^J|R9}%3@m-G$v;K zV;gJ%5ZVF~v?cpr3sV+LD&LlG@^$DFVOcv>8zDC=>bWSK`BIXkmZQ`C?3b$BBo0-# z4GQNAJ}%3&I|xkA<}rX^E?HGqc#+=M(R?sKw$KoV#h%AMX=r?BmfCm2q{>(^yz1JN ziju^X?*-hzvIYpdjO0I-*$ovNZZbxC#Gs$P9>U(xk!C@VjeP+F;7+z{C}DL?f&DI> z-01jF9c7P&>Ve?7<1YFOHNZ#F_{7WRGjofx8=g`_&MpA@)c+W=_s{<$&6r4e;U<#i zYGp58-*v*RFLfHP?oePj-N##7RTH9kyG2S*dre^HABvt)IUhjf3>P4K=wlWYD7$mljAb&xv{(|^369kOIUmrLK`-GDk8%}rZ z%Xgp|lwMs^Jk303@`aNV-V?s0fImMApvQu2`YZ|h>4|BKX}d51<&AlZuV<&KOJPmvT4*CJ=A z_8iludDZTp0WxPmm=Q>Ko?cq!?2e{yBMKz~Wcj_V+f0)lr;V2tgV3YaTYCKtO+i$$ zbgE{~-L{-x*MiTA?T?To_pKZ)%|+K(Kg2IV@om2Oz3k5MW*Yd1@LoqSYs?LhP#$!f zSt)WBPk1nxPPe-{&Y)ASJb2BJ1fBtZHMOdl_b&){Ip~ix(*Z9?TBDv#3&OmuEB>6r zyEP7)Q$k;sZ1b%ickqJ&S@vtiNPC-PYX;oy>08|vcM3>&WEo|1-|ZwGO6+8@DXheK zHWAxK{(=Y)*0SlFh_t?MM_XW9%y%E9K5f+c9Dg$0^=g0ODW}icbdy@oVvL{t6Mh z^?V;F?s>#D;J@urNSutDYfr;_W6Xe(9$?KEl4sLoVNf)H8^Y9m77WL0dKb27N2S6$ zj*&H{cT7c|eAT^CXY0G_8FPLSZxE+N$y3iGySz^X?Lgu16<-u`8w&D<$3XE7SQc4N zi-fP^C=t4O$LRUU>bTpD<^B08dinxn|7@-OWAOFis;fybAv z-juMgmrY(&T4v#EN>dwsm6VtIeMB8p9v2_0A4>rW$BWpL4CGEXP^lH%EP<3-7e0H| zB$uDxOxJ!cH~eY*Mk|61*AC!aEu7Di)o|HYFE^iG%k*yezaZ6lG69x45AQ5J1zqi; z(tKP#4!$(_r|{l4DCAZ~9Wk*RG{V{9jg2M71U(i{KGzD9Bd&agxtZ+gJ+fm79A*^T zyo)OL)xzcGc&fJE@xPWlSEJ#esWUkf%r8tPj4`Y@IX~|OJwgm!J#{?-$!nrN&J-Nz zBjl+XKLjjF=7_?tv?=(a9UL2RyznmTqBt_l|MJz4#8(p+9ElH(tmA2xrS!oVaJF@_ zUxNFoE!}5ewxb%5{C}@Xj4*;6qejH*+^t8yOnq8Obj(XU(TkK3)Bq`-MQzXp+}x_l zC8>>cI`7oDx{1xTTOq^sRC34S zg=_IldkN0SUXb}s)vT)Dy7)tQq;+ybcY$lHS<>PH2Xdv#6KJbu0d3U_5dVT7Oy3SG zE~&C%)JUc`p@wd3R}0U=^R0IO;M`98!LkeZDLFs(?yKST4!a z=OkKyQ7KS1kzwV*m*QsaB(}_&7jC$WWwMKa6guS_8b9Q%tE!Dq`neqZG`53!-#i)) z{R);%%=XAyEAi>L@Ys(PD&G$=*W&9N^j9icbz~UTmgURLp~M-5J)Hw>{JGdOc3#L^ zBxYjakQcsc2+rP7=}?3>IVa7f?(m}&@nW8SXD`pByxAu|c(SjxJ$8)|rD7RcSN4-B zg7?PRW*uS{PEC0s<_a}k^c>VfXPtj+Z>UW>a)R^nO+>Psyr!gX^|kz3Tia3mb7{+9`ryYLT-E_DHA2nUX!V)SX|@Z<4V3Eh|xw6JjJqPx2XcUrN${wIbXeG^CoA8s2B0xgTmj08CGX0 zS(0vq-TtJ4z2~DMK8$^yRmhs=quXQ+kPcD5D*2jqWDOwQB}>D%y6DeRn;!-!JZN^q z-dDn`xAV)I!X7Ue+EGSYIaL%l*j}#6we-!kd8y7$(#i*_<-o}$0w=fjFDExjlz{gZ z{Vf1{1V7b~@NEP7N(z+nH*5gAEchtj{?*&u{FQkll5a0e#LnweOOlevLR{$Csjxc+ zN9n^|`EPGFpSv*2Vez^?b%SBm1k>r88|1xwy_nbslv4kURYd_>;I9ZC%o{1k*a6$U|D|M?S-woQd3Qk%U!R^knVMJ z68{0tJrh|GB4Oc_`{CKSZ0g$^SKmGs>@J+jE$*vo&3?B@CotZ@VDf!O>g(?}T{8Nh7 z9B8jL045WeBB%VE|LF96lYR@FmxGX1?=x7lvNd$EVo!_ep!YQOAT}`Ts<=6ay@!#k z5$P|Z(2wg6;(K`2$Ni5Cxfi33Ihv0UA@fFULdt!ZHwAaNtfhL_ zhYdzQA^JtGzVZ zkBlpG)`Rn*!J1Qh>L|qhTt4Bg@i3X4%IAMlyy1MGBfk z^zqpRZ|k!o`1cl?I*0Fv6nTVug`^r z<2}bxmYF$Tv>Mq8>gVE7hTG|u=@?-c#9z^%2ypXn;fBzt*2(54#TV{(MCZ+&6b@F_ zG(L!-_+3&UR+ohyq)-aDC7+4t+pzS?vxU8~eyRRS8g`lr3e9Nxz!v&x+#oBG+CJ21 z0*n%W9<+vrF~q6vGTjq4bl-|vFCzEhi}1BVFHo~OFWod7)s}?ry&njg+zK>@_+IwV zj`U~Pn=HQjfm%(fN0@{&$sJxZ7PA0#98ZbJ&m&QirTxU^0E%)*jKC)VM zi4k8Z950#^Vy{_hd&A4E==F`us4Aflh!X7+>UtR4rQR;ZRzCR*pcV`7&<-RrVCNgo z5XJec`FO5pc~$Wi`<1OCNw4Fih6+s<4de=l{r;rY--4D@Cxj>X=w8$}+-LhF$09(g zbQ*Ddm7h$|ctSju#%AI5c=^uPxjPpsLtPk{tBS}@s8``=7hhT6$g9EK+A@SQi1nX! zxiu=w!}5xH)Fh!{aZxpP{S?*vmgF=bHqW!VJujzr?7-n3VNe zHEGVVX0n~}-pzhb^B13b&+O<&&(XpR;@42tH(^n(Ly%i}NvotN%}r~|S-FfGnAdgd(Ll`Mi? zOZ2GhHMD(f=uc0(!M=&*b}{4kh95$nT_(&N)0JhVd6>=~A_lziGe=L8R&h6TQ=p&1 z9U-vwK*8ra=b>lG&D8MKeREEYPe|OHH$}Tha0@HWm1*IKTKk1AZGVrYIpmv+J*L$#96KTnzy0T&;T}m z7zE4|I)r_a(2i(1KT<=r?M8oTV|Y%n6BgCLtDVzYUTVnuHf`i?81U;#5+^R(rjb1w zGHl6wuq-3U*$qs0EaApd8mbyYY_jru8{XV`i7?~$#v3JxTZ+C5uJj*e3Py@4xM<2&)f7=iM&ugT4?XN>*)A(n#lQ@8gJ$lI7R!n4}Y=xwik1OAoRd*)dWjb$e#+Lp$e(v$#w- z>1=wRMqI-pt8sDJI#xfm`oSt21s^#ZL)~V@T2q-y{kt zse>h?IGuVg!u0Gz#^~~^n38^`rFvSD))y-c>k|QT%gcO$Ees2ul?Yk$GWg5QV}{VA zo8|gqBDqx`oIagmv%J@2Z(z0-jd77$#v+$n3g23B=w%E{i@Qo%J?DXPOsyBn33kXU zG$Febn08+|)J2`wRtoHVL)jnmB}(tZ%EdQ8SaNjtqW!dL$lIaXt&d03ehKj`39kUn z2vYup$jsReF1NQsc+is->}kGJ3=8RQjTr1Sk|_XlBExQ}9j*}F_6{Nb`5PaP8Vj!fF4SbJq$|h|PxkymOz(d_6M{AH5 z4X5U_X@`6Mf{4Bwm6$-Sg?2P>Xot1%+d@;2HNq1I-$A>mryagPO_0E{-()op@%&w;&M;?GukOyaO2rk_Al=5s@u_!cef|Ah zE>Y(7kX@D&29|J+|pZqb9M}G|D@osAiTLVUwz^$3z+DIAMr(wx5 zHF%#4<-eQe@z_NRxaOaV9MlR{NYZ^`%HO|c4t|XpUD&Qo=PUCSY%u2Gt9t3=P4v6I z#kZ!dwEPF8O-B4=Yuc;?IUHN2&{#;}lFEokR6}al#KsjVwt$BvCk}O)?(1YyKt*i= zr?II0Zem`nt^rfta8~ohI9Joj$@NVYIS*&E9%X~o5W6!93W<#_XJ4&PT5LD zP+v(jwQlS7Bnsk`K{xVIS=9=gTf7Pw6gm{sauH`+IzMUdxvzUw`Q&l9tQh6|TlP;( z4z2D|L*KlTvf76NV%;F-S55mgiWq8^&iQ*^S|uepjV*J|s@X1IVBuAHsV9^(E&Apz zDx3o-Y!`N9p2O^hH?y=rkrk16c3tJFR?;-TTuLk_n(NZLSw=b~?q(zrYX2V#`}Ybh zmNsWmHNya-?(S>pmoD0Uf;HK6_tAh1ilp~sZr``Pdb0p0OeHBgyQD5G79T1F37~>l zuna+AS+ZmBR|+Ka=CdYwsRJc_s~xvYxMF0}(MtE;$n_#zfZ#L|jcTK(^rSJl+w$RK zodKF!nfD-)FENu!k?D=vvTV8~t>yaIpF3j^b@)xJecCcLq|WDDbi&1}wrO@jT9DI; zXy0DEQ_qiFPu=^s?#s0a(31y0x^$8?7$E2R#|nl{S;hZh1#Y*8ay<*%L!SDWYACn7 z51JK!+@(S(Kj&)xNu?zRcSE)mFMvCHA49F-dGWUb->JB2td=I^&OUm#^})9c^d<`_=x(Pk5B9SPJW^g{w!g#fBN=QxV1>lwEo1W& zSiB5RF`&wmGYr22F!u&3+*dsp_p;Rt_fEI2qUO~kETw|Ro8IW}*gy`%28c)hhz;Ho zXMQgmW2(+Ogv`G*khON%k2r}5PLuj?Y?b zNvU56U)S9x5BU}mK#fIS_8-io#L|89DCv`6z2TVE@A-Q5L8SBMy$>Mn9{H94lt+Ou zoBxnJ6$S`ENqC;m^>enHS&BJ&>N$?TGyD$b2HhRX_ZErpHOR(2o`(`tOQtt|KI-Ha zYmtl-RbLFF-ztlZD8UnJF|1-v#;&^Y^(m z88v9W#5aZ*Jc(;P;5ueHF*eUZR_8^Fv^mj;M?Gr}h4PIUTqgN-pkv3ni`&m4Hq`B9 z(`o48@)x7|bm}X|d^HM{Jjk|_uP1(Rx*wIT&Rxi?Nhi7djZvd#Y~1v>q7vSmS3@Ky*quOIBsTj@!_*wU2YHh9 znq|eWIru_fGS0PzMe)VX}FkZTj zD6r??XsR{t0d&L))7uaDq3Kf3>7`#%b{7`POM+k@xx8?)^BKMfs(x6eqr=N2{l21@ zB8+yRcq~0l;@b(`3a{Fo+5YXFz>Q?7qt3U7Yw)k;;WY|W1felEb~O z*C2zR-{|ddk;E$sF$%w2uzbV|e1CUr92B&OpcJgKt|*tzU5|w4&)Nc9%lT%<09(|z zI{wF^ci;FJ_~AKm%N_EttH4ul8eILL|57?9c~d1>PuP%YzMI!yH^AFq3Qh}W)@C2a!%7Mn$<1GvRp@^XMV`-->cNV zF#2Jt;yDu++u9#_A$i!lBU9{;0hZKtIX&ZcRB4&j$t!*rV)^xtg6 zL^Rb&A9eWLvx@eAag536Zu+h{I2*pXcBW%vTDav7k4CMnzRor$O~Yh#V)!2A4-pgw zwSE@TDk+J6{ipSUVFv1IXI;?e=Jc^&%C?qTDzu)pKhew0ZvzXJ*0MY_px19B(!r2)IqH#DHWw)&E0=ud&jN@99-iKdl4#3`-9WGV z4P#rmHJl#J%cTnd;g3zv(&)8Q}rqhNw9P!tlFy9JEr~DXi5ks!t8G9@xOt7w|HC!1m z!Hu-DI99y-2r;H_q;MZ)HQijm2&^5hMr}*j)s9XL#z{VukxPh$c%Gn_-%~>=kzpuZ zLKx-t@Gki?`u5nPy;DI$atixWPa1VZ#p^Olq{raXPa*I$>K6p#&lX{%-t3S;Lp%dn zi8_3`e_vSGP0Z*qv-#fXOSFqnk^?jUZ^%#}E0Dzm#ZDB)pQMkEM=+oyajLQ<2>J$` zP|6kSY5NCr(6k*^9BoF{-sV$d`VJfnpiXfk-|y8l4CjYfK1(coDp5b5^3@}8>S8nJ zyl88nS6h~X(OG2t&Ol(&H4CI0U+;I>VsQ@PGZ5xSt7_##BQK5V`=MN5pdkAkGOfEI zzII`1ioOQxt~9c)6_@MPMtj;*al^78nJkHHG0^)aZ?Scefn{kh>D22Pc+2=D8dzLl zz9t0Oy>$+sPF}nimy5QwzR6BaZWfB4f02~?j4>7RLaQxa&Gi#(A&uhq8=_WaL^*|~ zUTB$p7p!O$YJDPYVLQ0A%Az=>zEMWLV>4_8ZX;8?eu7?YqX{6_;>$*Wf>DWhz9+@w z(hwgKP-yTVKxKy3`*PfHW{k`Wwy;p7{N4Jtmzl*G(77 zE*{z`#cPUe*GXwID7F@|%yU1!HJ#kCKEM`Q((zQyKYsj>^S|cZpj;bs6{|6}6IXDW zEh9glZY{U{2@T;!27PS7+J=m76p5pZCjEEF4c&2yWHxXxFz%< z9|{00Vc}9!b?UcNbw6DV5zBplQHdbL5ZVMC~9xm|#vDD~? zz2E;@%XoaJ9(dy5Ko|tRj7$HH@9#@7NTfq?3CDWrcRRl-AXS%`96c3v+)a#3{bA7C*US5U(wl9VMC`(m)VNSq# z5*LM$5ULrSHjHsSpP1({Y@8jlE`ghcFTyWS&?&tK%}#LX2xgt?^c!cK6h^N zU2f&(yQ|7czKvK9x-?oM-+X#RT)ZgtM1Ypvg;d465sE^>4#Dd0-z9P>i!(iQ0Nm|j z;+tV_e%}R-Lbp8g=J5{C&L_j&`!z!LQT%unCfE6)&Hn9!9ngKj8c^8}69K=emdx&? zoF4I2eEx#x7FAZvIC>i18pIVJ?nW-w)z0%VDUSw>8 zt5@fl=jBJr*l><;}|Z#C|Yehv6S zzgkEHgbvDq6?(1!V6KP7LZ_`1vbm2mUf_a?F`-eMrEK)Mt}i zna`oWH0Pn`JkfCh-L2Wtw2SVMKYuz@Z~)uPlRwWf*{M|AD^lI%3)6to%{iHu#w|~z z`MWJEoK=d~ndCTb%6Artfghz%REnojwjxBX$rg;q%E?^V@lP56k<3WL z^=HFBFzhPxl>YUxJs6PFG#=gjUSIt&-M$`~=9~82@4+eB43<2{0*`g^W%Oq0O)Bcj zZ^(8(Iy@*B;6S2eA#j_X<3HT}FsbX}zIRGWa^iXfGWNDujdHNwHQCr$foH5+DanhU zJ?GlhdB&=XdzO=CyG~?|NT=G+Uq^^d2aO*R5}UG_)wkERUN!o#qA6@?wTpw~Ah%(`ujF3M+d7h<6y23r)HbLjDo2B_B|3nbNnHW4 zzXnuSYd+%LqnI~CyxyYihmOhx$$I7<-`U7_`bX7#j`{3u#S$!iDe{p~=pR-i*zM)) zCF?C`f~!&5{6H2gsSjP*ExZJ;lCW1MdN3yUpshm$WOg6`;~T6qtBc3a?nKbz2fv`pKrQzx*cR0EoPKu<4&xlf~x>Pxuz1c!mqe3f9uKuQ5AJlp<1I_Su|B2)x z%dajx_R|VJKkX9F{57B8(RLV3elrj4v?IWD$I{n3%v{X9ax!g&JNhqfh3l@ugpM)i z>@AJIr=55I*4^&kx!T!$8u@f1`$C_4m8DM4eLIbly`s}0(d2ai#F$?eA8b*e=prJr zI3tcEsNej&r4rqF`7UlgOjYwp(1Ywg8b%19+f9-GTq!x3;yOn?iPA(MfU)RYE4T0x>D{r8t2#&<~MtpXhdzB=sW=*|5$4)`&)>n??qDi*h zwr$Ia6f%6>@oo(dCb9%>bcOEw%ybWDy=8_Y0IVi<;V)4>q#AXeABXo+eU9z?hTOek z!;+yD^+ungZ7|3q7=Tt#7O>RH2%JX@lSW6BWS~v7sVCkK@EOnkdnB0m(wmr5OBN#b zbu~9m$&{Tqo4p_sDOwP)*dz_gb->2r3t89_$3UC@{*TI(6;ka&siQtAc~KGbbn2zE zdqG|7GYJ8O#Mh?hURN8$jH~T#1o*?o5bjc~2uz+S5S&;4vpIa14+ zPQo!>hA_I`XVta}r%>(Ocjji83bU3nd~%o#-nFQJ`%+3^Ko2zMC6M^QF7L&&F8-qE zTj@>i8(B!+=#hx*6Yl!aP0Ot`2m;F8Mds5R=>pZh1?&FzmlaZX*b-IW1w8}b8nik? zPhMyk3Tfe)sD?&Ud9J0~k4olm!o4cB?%PEx+!@eqg^sFT^Z$@wy+vC|7AB*jCZR1Y`Slq?fbVaKdhQovXi>L8=9`&gJ>+34vB$v8p)?`;>gzSVlp!^m9vCjc z@J|Z3nh^UNQYASQs_N*maKxz-`cq%ts7n9o0F8t3yx`qYS0*ELp&;}oZZNwhBC^OO zh~o=pQ&Oa^raTK#C1>XmtEiC|U)R&+|8$SN`zv+)u(v5}tA%fr+TV;5yiZ-3 zH#@<-DBbb4eC*im@~r9I3(YNnrVr}9|9X}AE^L*2oEpQi=Gb=`WiOHcd#Fec9GOJ zB1j&kM!x-~Yv+DB+?bA}5L>jGw6ffYQZp`k32~Rw;a(*12VWLo1&9 zRdO)oShEH3GoD%ui`$KJVBlUE%^MvVGvN7o?0%Aom`9JIjo-$gGM_t=nfL}fp=wsJ zmj3;@GkgAW|6}25pDXdJE*a8o+(N)EX*ga(NQuvwer1Sg{jcb`0;1*XMiP<=BPhLZ^4Tt@60VAOoBT7nF>P#NOq} z>=~xO9!?@X&%36fuVRXAmgGk$qDQ$uDextINe0Vdx6-78ruX($4k8b?>ZI3zb3oqv z|9qJL$G<6kpx#Tc2DJT;DMQpamzXx}j^Yo*9?&G-0XLm(XfB~^b7 z{}5Rw5~Z|vp6d*G@y!Azw1oygTj>r))miel>Mx3>E}GrQHZJRMoC9|&_qJtid=q*B zbF9rx>iM2rC0U>S`7yb8(mJr3qQ!A7`{MxE!pMjprG|~7=NUmW_MC=5wI+OY+pU_H zOG@I(h5D*YxYG&)=N|u^FW}pb_+GXP?fBfs@l9!1^OwZ`R z)n3+)f#$wQS|63A0!@g|E}U@*9ZhA#o8+aVqrAOC_eKYH?ue$m^qDC)oa-vD+D^!3 zs%H2hDTs>#9O0s*38zuf0<)Jidl!S%^rAIw`nYw(w63#N(X^wyfF|7a#ceXZKBf3q z685enN&qIne_1tjtP4C51Kv z)6n+c5dD#6cPjVlLg5j^4tm}jEE!I#mG7??ws_lcJb_!)DVRvft=AW$jp@XhlxDKN z<}eR597+*IfQ-a8;1_>%r;h85g0gNOEUiYI_{vWjSLx9uX(^dpHcw^YZbESv~=*}Gq_=OJwjE?wHcDFl>NXDXF%87)_ zMqAmkoCteP3x%cZJ;ar5>w`hRV)W@hZe_7sumwhtM?79=`Ap2N+S{%iJn*$EP}+!w z>&&<7PYMS(viBBl z?%_Z4nLJ>#UGTlKFd42=gy>?Gq;A<%~QPr z92q7*Cz@qYBms0%i(A@^ZL2yXz~}^dPa*SYDelEtEfsk7AbdGsL+#}#)U9`@_sE_XbM zi}FLqQV0r0*rw`>#=tiAurJC^U^OZB__Q`_>2Y*b^%1*hi`cpRc;^dO=nba09+K1h z!9Xxdb;MiW?Z%J>qROH!33lzf^S!G-^eSI?{Jcclg?2-b_VAoXPa3Gk@B#Ki=Ov>P zKh)($I1LCFv-wv(C??X_#>T1?U#9ba;>QO+1vcU~I4^@`{nJt*5bi+6Z;h^Zh92y}Pn=4nAIHmAdxjjwNHE$?kxBLRquYu5s|@ssL-dK!)v zK+ZYLG)FP$oYhOpW;zZ|Rky+e$G0TKAYtJ2U!rTvKRO7$?P*4&Tm=P}l z4^Q@}2DbdxDVxXuJyg~s%haR!k@W4>%C)tT9f!8o3_g;qkxnx*N;EIR5*{su0AsB5 z>fIRt47{Sg209mvzajY}@G_8FsZiv=S$YOb;)j9lY#rmVuytXnz3_UPVU4mYZx0?Wp_Jps9y%>??wXkN+W^}32mP}`7w30 zUD?GV&yXM&11dO2W?_R3+hN!p6IBp?9=N(6@l+FA8u2O%#QNeH z7ypRhf}d-YUUTv1wkD5PU9K7%-36kk`^8yjNw9hJNmt!sC-Vz8(L5>fW}-=Muh;LX zoMlq%S*42y3hpLH81!-lXkA$wqlK)+5qUvMB=-d)zJ7P9=T$Eo(;+wXDpZZazE+&o zA--JjLNPD!+wW$2S(sv$j7K!tSPUxFXxZ-2 zY=;e=RO`I^rA3LNhLaDll4-X-@pG-iFb`!!a?E~*GORs+SbIgl;mU~0 zyv?ye-PTRnc%vc=H|a$XO*Yp$44?9dIytL8;ubVs8>Di8xCr75poBg(EEnK%yROQ3I8r;8tN zx?Pn2x?cFN&})fP{Dxz6^ZvWvgOxvs_hJ{`w(vO8B0xn6as1QI9osA`-7Y=fP?4^* zRiHuW=lxfD7A*Q6fZc=W!|!u_1vvl)dr5O(3KAZ$(?w!Fq4%iS6D{52R62_4_?~{| z4UiW*)p-eL-5(gfY&RBe0tq@+Md8IKw5c*__QY5FSQwL$)nz0QNNjI`r|GADkO9e9XhE5DwmTckGZA z11G!BhuCdi--<^8)l^sqZ1?IKY`-IWK|^4PER#WD!nY3Lhrr`zfxpM8;lYVHG0T7t zdc8otyqv`=EdIR*;^<&I@I=`jD6ISAZi4M&A}!&E8CO1#vN(DvHD%J^#@fM=xK^^< zns&Q95k4HnCH!>AA3`t+?S^tt1icbSuiF5bCi)19dELSt^cfz+LnwPw>gqpJJ;1?q zMJRfl+bsXi>E&AdKJ1q7=Rnzinq3c&`_#YO&!N96FM+rz|9ITWyi4+~Uem`qiSrHN z^PZ7(;+O9xpm-pZ8qD1zaH20aARRY6enav`aBt^=dJl0^7s>QdRZFf-2i8v?Lmwk& zLSh#i(V^JNnV}GhBXg8@sWn5r+qv)(lMk(ry&u}HL~4{gq4(PAp$GS)3PVZV2DbL5 z*xg&+-{pyjF4(u()vgxV2EvKBk~Ww95Q7f5jtZ^6wYiFjQsNI-6-v{N zXs|SR_AfGVe=0Hd$cc8{9mKSr;#@Q?14Qvsb4=l6x7`Cb=TmmmI#M+Vu{RY2k=$NploF2jaq_2M71Qx-g5lx;fau0)yKx#x6=a)+WyVC<^?;4*AIbD(wd&Nsbx z#daQ#c8qA*e~$f)-osOZ#=4x7z>xc15q2S&ax?pYt11JHQ-XbJ_MPW! zOCvdYykF;u&r55F8}U0Wc&EI?jBzfhBcXsE3js5jG1Qxf*++q31odtD7|9N7BhgQ2 zA_o`>8tzAEkZOr$*k%Ib(L^}LT>F7)gz3nZX;1xf=slZXv!xWZqu1_FrMR5aYZ^-@ zhg|Jsrb|qs+p_+$={xn{)3VoYCnQkwZ`_T|__zUrdL5p<2TsgHOn>x|Vcm1b+9b}w zUbdocOP&KM_|Z%Gn?;uVLLD3zTI;XIyhLA>viZs-I&Dm!L5kg)`;#=>ackcL)x7Z#FJfB znhmg!zx=z0`v2vf{eVn1ME80SU}WI&sp@@r9HEMGILazhbzf3RIO{ao!I5P3;pm-s zo`k}O@Z);%uWo~cHMUJKl<6d9G=a`*euf^l%UlZ{oh#zX*PuS0?kNx|qrkJQa2Ie- zW=;E+<+E(3xQ8i~>%UYrqdd_!T^=#X7OudT`2oVc-e4`_4$5SZ19T?qZycHxiD#1!nI5V%?K(~eZjLh6kMkaRuV@)|@7 zb@;od)NTH~FS`AA(IMz#7fF7Ly49cICeb@T{(2B1`SJr~4q5ZltF;RZUL#^7saJ-; z1nZ_as26j2>W@Pdw?7BE6I6)KO$5=nYDw1{6I-QQ+sg_vnm#1}p3}E^zv9)iJ*eVP zUM2NS-`K!6NIM3)l+L(vqylcs{Ai?&C`@m1Oz3Hr~4@A@8y1^ zhDkx$ktOHW)kVgnK&O91ocHyue*Q`6M18&&da>t{HJ~bgN4$`Q1oElLzayU2Q8jG2 zXf2ya53W|*$SN%u?ZCUm+u7hyROVWfZ2s+5?J@CtHa~y(Ug|$X^Fr3Dwi^>FgVGz{ zly_6PAKynkdAuY>IW=zS%%8|mvAWLu$dd-&85B+()TT;c!W+TY$5P|`@o3V+Ei^Sk zO^SM?OJ(1KhEU|)7oew|8`v{O4359FHt!YWeYKoqsMedqxc={N_&>bE6afq7jrq9x zc~YpwmfpH{Jkg1jh5dBY%6$z!2;_YbFX@fh2+H^E^j8=7?OTr*qms|t+8(#vSKDx^!*LxYIxhqqXZ5|@9(Zuo zm@(aY(9Qwb>(cU=NV6bEzs&r~p5(d|wfxYX9vM}UlvnmfmPzfThANLYCNv@g=S1505+optCH4n zD9Mz15(LWrb~H&Y40t}|Fnx$#fdTHM5H(VY5pXA&sbUHL^&!N+KUDVrvk%o&8Tjhr zZ2YPW?zG4&%R-j$tyYHx~olEhaGD{6=>hvVJb!*@3-+10y)# zIst*qLm*|v^N!EuGZhwVBf?YjJ!5&0Y(VEdK?8+B?Lkf^X znUr5J$vj~OL{$eJV&=kUu`bU7R|}mS&q(olW7@Da;G=QxKbU8zYO;&#Wai6RzL70= zhlo!LB3Y)nD!vUB!Mc2elJKgFb@7c2qK`Pnw$IhUcS7lYL&j$Q+`YAe#sd@GlIBX2Lhq*meN zZ8oj**VXWfz2BSfG~tN8_`$*jSh&B#o6=8$%~HZ!9G=E195l_m^of(rV*f}@0mPj+ zL|`i7jH?Rd;ayuAWi`?-r*o^~HrF*S?-t(DIN|RVUkI~-_ra=h!A<&QNUYw(uG)p| zyOreA-x|K%7}e!1Nm4gsV_}QrQjrt?k#np7YZpL460TkD97EYSE|r%_GprsvI@Ifq@D&ZAIrrw zp*?6OYF`8DzmDa)1wB?u32-a0U0gxsRXa`>Nxd3g&f2Kg-RngChJdckO+bfR0BnC* z!1nhZ+jyY^vyJ2PGvqRAiYrtJ8b3cYN-zHl=%S626)EOpLBj3D<{LgQG|>+#jIK?y zYikSBR3%;KHS#BZI1ncaXi;$li!Bw^7>`CiM=t`x{0oiNzmt)|Qba*Y?%cYpxT?xB7g-lpu;Eiaas@cubTIE(#k?g)2Ec^F7to%KnTqj35K4W z53?>NUlD?J!x&TU++axRU<%vZIg{OyFc_?U1keZCo-a?S_im3f2H~%V*>^IZob;aGRl-2t5OWBldR;1k z?5r@cHx7#Y5x&2y;B7zb!A>_O;7zB~>KiYI!>;a}=kwoTUO`BGq5p=i9O~L~?9RGW z|AnII>>zSg4sYNR4q=GvYfa8-;#mp0L>;CI4>x%guUQLJevIxnstb^LoGYW6z{~E? zV*?R>qITR9T0SX@vXo$tE`9b*@QDnZeKngCxCRt2LDu^WF+N{~__#B|NI)v8d|_@f zrZcAW8(=A!Ij8Y>ypR}fW7R%?x9fh)xkOx{VmH4l4(kMr{u;8|BSuP|@>=*wVb}{~ zEsyoQFXWm$sRRaP7S)t^tZsZer{Ada_P&;`65nYcuQsa%hF;&aOI2->`NDh@2dOV< z2GQ+Cf8X8gzwhoJ3VxTt5|+lfVK?>Dq;tD)dDshnu422diHTs4cO4A!-i=G1FNo*X zOO|h2wiE38Do3+12F^5;#}f0TZ|7}fvT@kWic!JQcPY8gA75}5{*>|XMXRTl|NXfL zY*oVZJp}I>j|0}R@UtEINowR~dDY|CO?C^BC_&a9OeRH}S9_8Fwy=qoMbUK=E}`%8 zrTBVo>P6(O3#qWli9(2Dh)fb%400s#FZk*h@6>yV(u)m^DHm$P0ZDTec>E<9p!5m3 zOS6@W#ouAADC`1bnw+NX1AXakyl#fY@hbTD- zaeHLVNyi3+YN8A@m>h(YbcJ_x3x@v}RrtTlsn{KjS{g>(ZJS1aI#f44`x)$p2Hwy?5KNloPzB#Us+6o$~5sDwS1Y9UdHvaA=e{wQ#n8p4pjQHWYv!hrk zig+nvb=(Kb*=ftt%(~>CLKTgVAB+W`#j7l~{8`!Aphvv}ILYiF)omccrgZtK>}&b= z!3;U8G%c7RzuHUDfySZkYCwHZr%AQGOX%Yplkv57e7=*HWNuB&iQURKi}<43_9+qf z1sCZ8tLBVdZpqw~eEP~wKbg(*#$aFlJZs%U@?wM0H?7Oe?FmuKLTB;EEt#88uaN@H znyA+6bfH(Zjh$#Wj6$%`YT%tsB969Gg{mr$+v5}5ZpK}<-ddifES@LtsMlc%i+}wh zI%v7a3vi3&h5Nsac1bC3jtvpOqx)DoA&*I@{hm ztE&Wom*57_wa27~DHj)X@7L8GFK_KD06t@WbANgy3Fy^SP}v5T0&p8Q3Isg_hX~Nt z?8#bSI2q{nWWra#>_FK2FSm8&;G6NxH`6Hu47ga^4to;3h~o~eD@%&7x4L7jW&HG91#NntvyI(YCS70wBhdLG{5d*Id zbm_iTH+nNcT?Lu0bo!}^X*ndm>Cm>Red&fAklVEc8|#B5-?{iRvDYcZ2_ZaYx$*Lj zJ~khK-mO}24)fSgtlsUUF^B(HvMxwCV4H5j!cUX zflh{j5u)zDKvxHpJBu4Xu#AI=eE!eT zTk+%HiWrcBfyGN%H;38EDVN7VWl8a~MB}be@6iylD#^D=EmnpryTxi^v#s?zw&@&+ z;{lIt+C}kYx%1mwjEhp|ko$(iC{4!)BGu%BlN04!lL~5FEAiU=;wzCR+7G?O)}Q=w z{T6Q_odA>|F4X06vvSn4c}H6Mi@`9>|03J`AN-7e|Gbl66#i6-eX`wsQS!=^aRlct zA+t0UUalTBe>h0g=C(3fG?Gwx8$sk@l!_6awBaT3V-w4V2O@5aoZ(yC-BZ5E7znhD)-jeLM5E ztbr_AN!W*UN(Tux9vOB;kTr@Pse1tt*%00wQ7JOsxmn!QX3x_zEM|05t*bp)3Aa)mVjhM|Xykbpc`H8glRQz*d7U&P_`!vIs_`=PtfPqh;1$nZMn z{hG*nJi znuuF)s1)ceMD7gi^e$LQXZA{`=OwXT-lE$0F6iuxwBO-P;T&H|y*m|J@ zkz76d-)kuP3(dS8L~+MO#%oi?HQKY~gY`7cnyx^yg8pBZzxL1Ndn*^b%|qU7ziU0) zdm=M;4Zjk9Eze2uG-O@372>H2@pOgr;ZlRfU^_Sd;2V$U4)3x7`?W>SeqaA4Wa7~Jl}qKpo1C?**S!!*%%ao*mpAP2Fbyaa$Yb#RR6PV}IBiNf z1BfswSbc-^hTz@~Wy-Iromj5V+F8*rtxgy3DH;tp)(SDA%>qm%j}C3}9VGkf>y@|{ ztf2-?rqBbHCNyyXG$}Bk&1UloyZdl_9%!s2=C1pg52m@jUxQ1}m`>*;UCvc9p{MMw zGoL7fXHtg{JDL=6&~YvVm*TRk2RVQfSAi(LD0L1?oNc*}`G9?lK1f1ukwHqfv10ze zh<$&~n*6-~zP@Pr4LM3fFNXu7&MbQOgE{FKRs&`~ba4L9EKF_okyvW5+8q$fnb~n@ zXV@`k<>r~tOhbwVHuZ7If_B6WL|IRi}bxnqk!CLlz90SEyC8%(9& z5FqlkVJ)fw&7Fl2;Ljx0`7K=qgt8{d?Y+h>ut6oTLC>ofRqKdjj4 z@YGmNPBk`~U1u`%cN7c($MYxjF&EBcs~Hl)AZgJ-O6UMN-Lae{-U-+Ta|g$PSYzW9 z`iBgBbeHBc(xSD63tu{b6A&1dC=q?_qndb;tIR9*5UvmAOH6{W@0_#Fj|z|OHF8On zEURe?53)?FN(~F_tlGX70)Nn}vO9QE_jSr!cqMup>zOhU>r>ke5dh{f*ooC1B zQ6eh6f`1pE}XwC8ocKh?MLxo;wl6$v@# zE2Z1-5LYc`-*t^hdA_n`Qu&qT`m3+Gn0OPc+Fo$FT7wS3c@3xy@RHbm)}&v=f79yA zBX@VblLO!+x#tL3!kA<7P*?FL2}Ks`JdP2&%bk>bJXe)iwsP3q>dfFv8&f8$vw>?qRoh@E<=zpdEC$ zmksuS`9V@JOfQ#E7$Gl3qr2$mzLnjcK;xJH(*8!)nMci2qK^kybX(~!w15uMKhM}= zj&khJGagyX-g&5uIYKcXNyfL>$NZ)E1Z_Xunn)8RX6~kbE?=z(7r4;={*0v~eQZdAm$f9B6|l`TC$O7r zbBEW&KqncnGJ3{K05xNI?vOi&xfXSvd=n|uad=dgNb7PLveY}BCCAOiung8I zRmb=|_{TpQVweNKI0WlP9|2e^hxY%pgU@Ktl>^_cVPgck`{i%QPw0vUR0dhO2y5*{ z>AsiW4__X?(zGbjTwA3T@9?wlR!m|RX%(a9nnzMv!aVbxy4DIUM@A4k!Fc^YI{1|A36QM(PlVDU?+>Qt`YTol^R|n!6}l;g>d!_jTp9w$ z-icd~b@Nr9#oODWA+EpBi#~6C-&3v=*!K5XEN@ADk5!FJVz>N|kOMg#xv+9Ho4FKs zI)LH1+jsIK@s?|oPb`nAKLdbV!5;`psN!9!C%Zej6N9gVHT3_IbhTd;EWz3F7+yUw2?u8K*g?4H%! zH;W0?*fczj>MWX@b;0d(PR4blDJ0kXIiyq!cGm;R<{%Cf=^z5LF4? zUOU~bG2}+(m@H0L-sghOn0K|w54uP*!S?Do(OX#84RTS7xZ2BFnd;K&60J)ysP|@` z9nAia4M~`Tl?TB<(0jsLK-WK42iaP9w=6;QN0pNAu7Pr?zqt7M-!j{M~Yf z7=WIL>}n1q7w0RgY6S)=X{q|AnP2y8jagjvf=>@6TaMV-R=A{!wZC_LMt^`)AtVu3 z4N36Dm`3j9+WP2ETJffX+=j|wh&MYrJ3S*hoip)f&wD{taA!-(&7D1c@H_Xo|EoO4 zguXZ4@6F70gc=%aLwcq;bnoj5)Lm?mJc^epTO{R5zJ#+#W)SZ*eT+D;EW4JZrey%U z1Do}>jY*}8H7!~C5s`Q6LS@hQ>WpaJyZr+Ku?2#cHe;JtL`(sj$3CUuf?kPHQ1EyI zr5R6r>s{cQ%GrS0v9P)>)8yIFX=WI}b95u;Sh=d(!W(8TlEv*W@*bjJF~&l#-nd>q z!)*#R?kmg)i#@BQ>4r-W8c`0PeT~1{fm06bNZqG7AFIpjW`j|6YK3~+!gwGAX|nW> zWfb<@=C5^$QBc?dUD?wl?Ociq^-D{|_-{x}H3Mc^e&kY|P^``7N$v=*P+j#`-AW{( z7Nd(TDM!z%8L%Wh?<18hmZN9iPEq3KL|+NtpGmM}0bX$g#AbP+%k24bOg6K-`VcB)6!u0NS* zJKPTVj8PM@^~(%$bP0B=Ca|qsv=1m5wXDYE_Q_nP&Q~GR$;YqA>7FKd- zKPwXy4ENQ5iczP3Lw*VWhD_~Kr*M7Md+;ql$dEBQ!lJjDQN{n7jfa1M(V-pthuE*X zmLY3DR}70FWO(gW9O|q;Q$JPVOpVDpB$a%|h+U{$@_*fz2rmv2Ni&`eMgF zOLt?#oRq_yB6yUf`~23d%+z`4RRe`5^j}|Vp~d3KM7>P6S6@jjN`+AGiyx(n4nHPF zQHs|oK%A4L-3hBqOOR7cw{UfJvlg-H_^MkU`BCe1#}N%icx-p$^OYTa1HB1Xi&woS z(GSI1!{Q^bcRVb+<0uRZ+q}+Ou8WN7;_+9LX=+S|K70f+I^R>+H?X`xr`NGeD}gtL z-x%XJlic)QU{$$cVtp`!_RzpnkOTJrt^3Hn57K#%90`lI!-&5C2kZ{6_5c3a7y1vt z6NYAII~wMIVAjXX=|5Of+f%gQ}47bD-Zo$c9q zpD22NE&NKIklJ;M3hqrCS;h(N6s1pKn$^5Qd%^r&<(XajK}A@|AzYIDV4d?0<(|dm z$07mx0gCCD^!eWdY*V11Nz!b1d^oxyqnI`n%oIhEwihBo7NsJLky>UC%Ri0|^9(Ne z**zXEy6@9_5HZpk_BHEzn*b|lZuGtbgclv4jnN9=tEIgnLb1g-lxT+c?noPb6L&-3 zv}}3SONL9tmTRBxL-wZ@V<1xWWiAxPB$pj&&6mX}Ypk5#xeUCof3%uiTfTIkE*!!- zous>(BM^s>qjsd>ynx+OP4cw9NNE-R)ddF;I|;s;;VDJ--D(K?@tLjCthwsxwiA@C zUk@2Fyt{(l6=FirE(SM#ZZB!oyL)*Ee>C@#ZsUcf%3sYrn_DYkv;;uJ&SK}0p7PH3 z=BlwzK2%;7KBP%+M5LP3vYK0Vdbo!MlJP=fXvWz~G08Es9cMQ8mMtJUNskjT_G3xenTeeHEeCtrPJo^EJ=s z)f_g=YH>KLmL1Bg!cbSe`X{U$oaW4(?&a;O z&t1J7FQ*H3yQV;jgj!@g_Hvl3%c>c~b?JQJK6oYG&|a}3AspT4pQvE}zeNSyYGgO> zA9IG7ATige=b}#Dr%IP(_%`GC!a*q6vW!T3KeJ{Y*G$ax5UE~ie10Kas5&w71|uZ! zfKa@^kxjTU*;#;1GZ=f+d(N^X=>MVYy`$lLzwY4?l0kG)gOMO2L<`6(IQ%u zDA6K_UPp=Ef*?T*K}eJ#dL6x_=q zXP>=oZ_au@dJWkn>2>Y!79oh=N9@boB?48;Q+Iz&Wr*`LIOYCjr_lJhyQe_2zHBSS zOf-`HaSNf@=K{wi(rsKYdAB?$d8y()z;H$|0uyNPt)FCqQHuCreiX@_lbE6au`BwAOB7?uqu-agt#I>QnfifzopcEbd-`Q8m}5R|7+GVlka`!@18cc zD{~nM3BYN_9xYDrEikbU5FdSw+UBdNai~@FzRFJ$LAnO0b8zh5p=-ZHJToBV$cw}q zGQSpP7$i@*-kZyG9zLK@)|Iib)zkWV2MJnF7a%iC`e z%%~1?N2W=&+#J019ce+qv@u&%lI<}AACy?GdH+ZC3twwn{UU?J_ph&f-hBgJD<>PL zPU?Uw|0$kG#*ZHj4bU4~MK#Nw7Hfi`p)lbCip82*-#6@rSA`}`-dmkPu8_4l zsH)pKtd}b0L-sk5SDnffefk*TaXZdyA+vVP^yvS)tKcVwN{yeyO`?Xb7-1P12)VCJ((J~C`o&L9A5MqC(yu9e(=h4cWmBtj?3>1#J zRbfKm=CxE2zwa97$ZsH(*CIa`z*MahN_~Us_rA{=p)BY#e&%3P;GbLQ(@4V>k_hmHW zK(r>{qXAu@c;}1&ipMBa{sA&NEdx=Whl(pMa{N3}^6mj6ZwE({!$T{`uFBI=ohRZX z{2YWy&^(0#U%@aZ!n;avD7umCpud9(mCj2*^5+hNQS{v=`VP)|QQ-^BvT@>~o<4^t z(orIbpW=|aZ=r@G!{Y}t{Oz94c5n3PmqUJ$pKygW1)rtL-`pU|#TyP0GRHyRj+3~;C1qmS&)RCpX^*C ztx9bIUs~%AK^wQzB5eq7fF6zsy+w@^mtII9JV?4FYI$|sIMkU|DWaw2ko$COii>hI zy*+?#pfW$y-1k(i#E8$a<#NHjzJLmn%BT00%qPACeO<%Vbsl*PHkY(r)5v~~>_rVI zz>uKfdLiKYq-oy;uQ$G-u|Fs*QpDp(I+uQuhK#)v9E975gv^Nc6Zs7sX1Ir*e6-Hz zE%IB@8M@Ps3)RZqpH9@wZ}j&D{XyT;2Y#HmI22AqXLO%avQqZ8l2u5LJ2l(Wr?*-n zh}&KVD7m4giv*LK&n0W0s~KUZEpAWY)wAg7)f;n2I0gv%wiDc^Z@XN?+e?i?xd#Ei8L z`O)H|IexQZJBH@ov0250Y|Go9xX!0`6zdz|8H@1cdKJM7PY3+QpBH*NxD9m1aRiFy z?y!-@1gqQ$+Iv4+!QVcb{O7((S)tf9=GO60w$(dq_kfJ|f!QD5dCjPXI|~l6uI(Ps zC)H7ZvecH$O`rL_5!EE++=#}bv8tVW1l2KAg+JecEa!u5$0&bLqamaOzTLOpE3S)AsQaxGn1ZW z){(n*;{Gc|ox!+6%=K!r_#GL>6rOvm5c6{#B7u2yAev6ZeM8XqH^XE3=|wK<;q6c! z?H+yy#>xC*|2;EGyv<=0KEw_+{`;GK!g|VetsCWuqpWYgdGYvgTi;)VX^E3}!`-z= zo@;g@FVA@WoR6!`wu@(6oK3Bmh8H;Pe2e#)IP71zrR} z&$nwcA3M~ZNjGxfZ`q!LFgKiIE>qaNT>Y!aI9P$ha&I4%TL6$+yk{r6LL?s9eiDE9 z)kY^EpDHV@h}toCLDW%PDJ(J+Fi>%ug1*Yn})BdBRp1qv%q$8Kb;q@c= zK^G$8*AFx{#)K-v>L(YJC})}CNtlgpXI&CEi)um0g)r}M%wDY*OzYQZDzCgm0&ok^Z#&oJ}JJDH<{jyF? zVU>)@lrG_p=gW(*;ik2$M9)d?-Q8T8{}ix->5X*HpPZUhvKC8R_lyZOXnoiHMy6C2 z!Eu;H+h_|Q^~Fkbq&!W3W%8iA6W#cvJynZe%9}@t&ZKmr=t!oK4xlfp&M4S9C;mMn znD&9-UOL7mv_bjB!n8fL>XRnlkHU)y1}cNzQNJF&Ym=TkSMF%~_|10qa)Vs{#?R~P zqLY}13Dx2k+P@(`C$nzFcx2I(>y%XuZ`yoE2g^nq_IC;C2=VYu9Y_15OR}H9h38xFlIK~~pXf|Hv5rH!b8ePDSyv*TlIoyT7xD?P zsr0V|=Wg-o69PX@Khn>Yxqg0pC!+3%_$Y@H&@ ze#8;4sp}aq#J_L~;|iw_3E5Q4qRkEv^b;AabPDug33IE8TG1Q$i&4U%YvO**CpN82 z06weu3oaw^Ixt{-_p||MEhT{SRB!*NgEpGIBoZt55ce|4k&OLI#kEKZ+0l{Q$f+8# z7ZKwM(3p&j1?ch)-WDDSfcHC~!-4|b-z-}2_^kmh8bFR+E*6QHW3HBo>3%kz(O{n* zW)LSm-S5G5u+4-ycLl5OQPA>tV94;hJMtE4)06eW^ zsgfKxTk^MT@Awfv^?L^97+ zw)a74K*@&vGH~0bnmxYea4y5X1~kr?B~6^>URPrpEk>`#YCPQg{3hIDniZo8bmDb9 z0~HoPbO|Kc;{e`LP1n14xUn6>>FQS#)qYgh$QF(Pz38_@LF`!E;+UVRx!a_#uV1)25fBEEyi+u5>}J2R5&u<Y$l!*|4xxL!2$Af;JCh zSqPmHEmrdPsu`q-dtIinIBzua2GU6|G;w{tKUZTqf=c~_^7IhwC=}^g0a0LsNNNIm zf^APQnwk~-%{O_y`^PthiEqs)>2<}mbLN-=;Rzr#rvnV)HUbZf1;MD7vl%JD(t80F zF5^-8wT(l+#8ki!UAGzYw3M?vW}(uv0;1n4FO6qf==KBCC?U$J+7*eODS{t3rRvIV z4}73gaWV7MZzJA6%f9dDXZW=%XFXvpMeNCdvLk1acqpaEVtKa36LDYH<|`zyHjyBr zG}rwBVM#fz$5Oq6>#|>MKJ}OIIFFO4^JbXq=MqQYm;E#`vsdhVcEXC{vf2L>zb{p3 zF%>V_hAnf`c`mHWZ3N-%aG*uw>b0U$F?w+GK;!by4_j72TiMZ9$A#y*0~c-NTGrw< zRnPK?!ua%-0G=T?k;|G)LB4toDH}i3O^#%<6iV_Gu2S^rCJaHjxj^HGQV9b;qpU_A z?ZSZ=>KtBkz8*zRknP{U{XlK8zC7erF-)+g%mP|+*HV52_lDSyU?d^Kr^jv)!* zX7+xfIH4I)EORW%#JJ1J#h1d4CnhmFgy~-2JzgssmkcYwz5Me3!@0b0-unDa8!0t3 z0p_XOT}+gO7-CgC{#<8z#YO>zmXX0av#^{MM*))kFDk>-mmF5r)P~DB?(qk0nd$f` zpks`do*EIxMv`o~SrLXjb&vFpCrtl>{MkibPZSOp*{+dXutX)wPtJgDN&&8c7Cq8Wz zr+lk>#-N*fm$uUH*ao*V-|neZN4Q%RSooD8ud-L`Qu| z*e-HnqE35A?hxiK^iBNU9AnbZ6_P1!Gk`9%8QRY73)#c)Kq1Ps5<7+oh+IHlHraTV zY5jS8`_$^r8*>O_M%+JM)r_{;z1e_~G%A+z4#)X370fh8k*oK&`eClCWGS>-pmogV~y?@N&ELAS&l@58R%f4J8s9ACV8m87ceS+-^FA)*;tCthe46G*~y3AoObCI*G`9hOPJ^mQGwY}_we>PhF z`inRcG$#l~-4Usc-Xr zWVrWP=hvnh;IwE~2IM>wM$Vhj+Eu$uuT9V-*|~FhJieIWQU|O!nysuD;OtBCB7Oz~ zM79#JXxkN9W~94DnLPba(0{z5*2Y}umHG`FmCY>3LSpf`Vkksvror1di!Al^t{eN}8z zimo7;n0ZNMbSJ2RzDpMv#I!UCS7(lO$TYAx_fV~Q-n3oPDavbukZG)?KLdB-R45VBq^zoB7YRv zEmXEXCS4N}pRWE@mhn{=IA^<*L93pU#@!?}joO`!fwAqIg{6*quy~}hhh;lwx&$*{ zUg}CDW$;%6skf;QvNA5h;1YK9Gs}IX91R?x+*&%LPT`V;FnU`Lqo!nmw9Z{Ip(l!D zA?vhC8Bv3v0@;`Jd@9Sr_VQLk@8!1RZQ~V=Kk=B%L5b%?Fobq82t>n~OuX9-AoCOqEbpiK^14un6sc%|Dq&P&I@#FSsFdmWfuDPBS0` z8$Q=1`iOCS)7Iw0v|L>Hq||Vt)8tRwomJ>p{=l0TdhD&t0w-?k5kDtPbZWw*pOM3YA;XT`xGmEa zP`n8_z`wtGW#xFagC>eZfOF)_uBN%6S~G##>nN^?96#IA-}rkbO=Oeq!kCdOTY?ww z_H(F~*iRy z!R(6tgyV9;*XL^|pDI$StG0oP!X-8Xt&5M&+5dpCbod;gag0FJ!255=;8%ya|M{IQ z>Bv9I6jY`=V0opsrFoQj+Puh_wP(h5YYQhT$69ykkg#>S#*88X#5Wk|L|wKc~Pw3`k@Am`3>KC z7tWc~0^@M)>(*AWT@UihpxH$`_y_j^?ERoA9FTwk3lJwk0&w5bvkKg5=B*kpD`Rh) zYJbR-P0mXzTRC{j2H2?2`Mbl-GDK6ebPNg>?ND1vxZ2Mjp;qWopvmEim`29lT0z z81g;k)B66UUsExI!)I=!kanV4A@hX1#En{I2I`CwOTPM(Y-T@|=hP(Q*JE`<_deN8xoBsIPX;P*)BUq{%BQV$MEpUATC9 zr+}<0P%Mx#Ax;#aEwAbHBDjO^agS9KyP1c6;@tV;*i2bpec9vHar^C^RlTU zwIzT#O*PSpdNrP>?B&a{h!p;>{7yGeYwR~tb9f_j)~cRST)Q|rdND$blVakME$M^v z954}(>Q3$AzRvq*lOb&kZp40>@CXyi751@#Wa$4qI7$8rY%q(zg8F<5bvu&Eq9)Sn zfvQp%*)vrFHxYEmvMnE+XJBR^19OId|I~0edr2a%CQR+)QkpV@fXNRlUeP|w4O6ti z%#AS#G}Oyq_w;dnKv@}LI220ko!XGW=CMKdd-N?iL0zQ z%oH8XstZ%Qq0!^1dOg|txMz{eVS6?NPGMDC1;n}h`Ou0ZFx@A{R3!eG|1i0JGggDM z@Oq-NED)<|ZFD*uuKEvWW?6l@^eSxy>ZG@wIE>*`Qj&DDUxmHH~0# zHFL2(vv$(Cs581Vq~l5@UUJ^HBs-B!xy&L_;8}|Y_uL_Ouv`R+$}HZDCWMM0R>5(= zd`LhnBpM5f!{_U zaYUZ8d_>r>sKWSk(ktWh^P;F=EqKHQap-7u07t+8J8cX>+bO#$-SbPI>u&RIO@hg( z2x-jTZ-}oW(;|#HK-LyPE4et3TUI-aYNg;;GvIzpM;WZCqu9rJuS9(Eu(aP4!;5UI zd#`;5vP+Rr6|?Iy3j|a4lbMV)&{ZQ6laC*?YD<2a61Xo8QE;=S%13GeKiTuTDI z*|B$*@35PK?f%JA1Z@DFpFl`Q)m?V|=A zDGbP@@L(&q%EBL4KV|;H7*=(&UhiGz9D{r~(Bi8#dlK-|Z1+*X4a6Ae(+@D{17QTJ zym|Nu^#=NA*o>g;6W`VsRtF98lXU&OmLS)D*SBfxb_>-JBv^DDVtzpdLUbcJ?i3sb z(96!Ta3<@?)ZdUbiOQX^+3#I%g`I5DI5iEc;#7r7J8{V|j7KV38MLloW59ys+3*Mq zY^zXU$7yGP^1+d{X3{$)CQ2wTgSamvkm9)+Fo@VSCg=h}E6CXf#KzLbX8wjSG?4;L zx6ErYBU4ACEQ2=<(+1vAM9kf}q}xr(#`GI#r1XUpi)7Uxbw-63V%sVh8eZ^1&W7GX zVM+>IBlGZMrV6K%UGQ2Jm{Rs8+Q|oGyF29DRsHz7+;_c$!!lGmDNvTgzzqAn zffg_!c&UW0L z%vq7M!^;Z&J^!t#zSXE?cYopoXhbMiMjphQb`t;)|DpBgGWLOR3oka`7tS8roBc(@ z+uL8<+rqa>XCU9XUV#IL%=2*v0*i`g)#ycA?AV!i4>;?hPEKlQqh=}hVUb!!)GtbI zZ#$~*ZaaQ}Qu?5{gF%0ju#PGaLN9hX0lsc22>AOYc0*G5Aga8XtAg-30UU zfQPTN1h;ciWbdY3?{ekXo<;7wJ}w6jKJ8%NY4b5y{WJA|K|Oo+EqN7ssEbX7<7CDV zJ2hzF(+F!tz0HK-Ux~m}ebdwHiv*{p>7H|U(m6s%5;8x+!Ia9w&$V0nokXUAo!svu z4eB|DrmC^n=QCYbOH@>zX2c1c+>L3tQ#|9Jq=KVKK)6>_Cpma?pn`_Cbe@2L8u+=G zU&p`1{JxI`kFA<@IE6HE;*;F%yuUS0=bhCZTP%;fzthz-J{d%IL7l@UY2!J2z>fg> zXJi{P8xS9sXy`7GQHK55AgZB5;TvFmt~#1VK9%6S-laIOCDC~6Vs%+%jOrB!0=M4D z|BCK?+IIK?w{LnIdxhQo*PoG!WV%O1e_qBo#~-`k^}p*cBLeSR7J6N6PdVugziGH| zY!!Q~o>V8g5C5<(jY1Py{FDZou6udNX{fzZbMTt$XJ`Bu%-&PC?McN>XCDCwqVgIv3zbv2-1o=A3v@jr;0y$1A%nWK zWwEPV1&Qi+#6c%C@>mNxkizV7;{Tr6!$nA{pF8f}*yjS)3#9?PUg<=-9Wwg~Y~XXb z_p+nco!KHGHj_8%jIvRyhcd)?&%E{0+a1ZnFQqMCi_`AP@00AE^T;RoO>iRdLT;K2 z25UbZ3#6k2Zu7X==E6OtU;a4U^zQZQ@K-f8#IDSFhTf+@$(QB(muw#;TsFD)0wN8y zh4>suORl?-EStgP`5t4(THl!(7u@z58%g^L*y)J-*EHQV!@L9UiGSiEJL{cGX?8J_BYZ8nT6; z(C79B$JgZ9{>9;M`k!9~$~`;JYknM~N71L9b+bSxPmcgQ*U39)93MPuJQekleL5

<@IXqCOOT0 z3x$qH?V}^m)NZt#c>76}VT`Yrqu8OkV=3x2g!J+|6>^Aqyyhba0fVZ9TcD2l(okn% z)*!Ob=^^meA{2;zCqRR30Z};ZUvDo&{zz1KB?e`KBJIO9aAR)|j8}^) z(2Ta#sfmd^!83=_yl*Z&xx$f20uP0_x#>|m7!FCu@^}{8GpG5N%r&_W|75V*XS2BE zKuToR4+5z@PkcDyUvZ10Z%YZ_--7L+65IkLJZ=4zvw+Nv3HfBq4)xSpT88$FojwlI ze3z>BqM_P0ITiUf35pK>9r^Y<0^<(In6ugl+!c})c$q5U{bL7W3kVK5pHbz4y~Hbi zyyxwUXN~D<4(WxHsD$m3s8j>I?4^DP8{1kR7m({W?rdTi*vb6)49693=@rNvynJ|X zT9J)SM{!Mo^P->LDNu#)c{&J)g48U;_{^s20fO~>tUcDqmUwMYWV>7FT2(`wD!04r z)1;`S06_u_yVG76tx;^aT`W?Z`qMc5nbsDU`ai<}XQyT{B*1%Y)n;=E+_7#e`e6HS z$cf}eAYk_BU{lX79n7Atix96j&bFOMhKKJPKNeK0Huk;za2FHlJpum;%y@Mb*qJUK z5xg~t3y52p3BxktnQzVep-XlfCOgee$3mu{;6N=BRT?KdV3z`D^CO%>`(m>Pmc(h}QBKF1zlSCL|oM0J}G#O7tN%0DyI zeM99NPOgnIA9{%9GN<5*MB76sBXTIAr4Y)p|FD0$cMJeacfp7T!lM6gXVK%K8AmDZ9xOcZa?P3;gNdC z;z{euQ7qH0awzn)kSiA@FX!qfI9Ho8+8EFZQc5} zV8MU7QvD}OMUp+*t&aPJx%fvH6WYZezu956to-;)l!CK+YWxIgbbO;}`|OK6n?0J| zjZSs?g*vwB6Eah=Ez}bdlA@P7=Z3qS?qw~Z5fbS_eF1--7ohO zK&x9P404*4pjczOh<9~n+Z@@~ti7dwRl;XB-& z)_AITP;1yv3NOCmV0%ix&;8Cn5~Ym`-<(}=!8j^ccoj=trnmt-mG9&#hj7dlas=Or zZ$ZrdRE@< z^EN4z|3|f13ajEeJphn1^GSKfZY#jPCGRIJ~P(l-`7i#Wy9W-zpDy zjr`;n$u!;m_`^!FX2Bc5a=hGvcLP1{vgT;6Hy2QU+fstfJe-p9Ug4n61crmFBTN@V zmzuJzFHXPjk*rwdje96fm1{+>`_CtE{uF?wUNl}}HY|X@w%($QTHbc|lDFGUbRp&X z#xm_BOBYW01vEtBII<==1+$;yKLT^1p$*8T2hzNJ)kcqWN8^Se$^Qqu=YRSe0siwG zy2@?04g0OdGUcJVyO)kK7$G~>m16Z|)4M0vELVszI)k2N`;q$N7@JA|8oUuFdup89 z7|J{U@>X2^T?CC%5>iXYUDLF$zE71nPL;v@n8ralGWu(&5}tGN39~f1O$O zX|mrtS;nAa%9A$@PKUO5n{NaeEV01-_!`SeOhM<&b=C)8Yp6=+l4>$#T=2ep|FNc- zx(F6&8i-I+p%e-(0o}(Lg*i;OJh&8@#-EAAR@}juk#DebXBelS8s2Wt^L#axYXii; z_xL`f+(+tY>No(xl-1QpOMD}TfMAiU;qT>}7Nq7E#ARTeg}+bl`aLvUd-G z=CF6o67Go5d-F|t69S=Wwy3>oxL8#i8JPXT~oq^h=+=%@Yc~gn2t7e`2h%~bw{W6}@Wc{#oy>!CvkOt%jwWkjr zS(+|#OIwc9>h_SPK{gPM$IHZVYy+3&G1tSsa+k^aH!%xe{5O`_PhAhw@p{DsMRvKP za8nB5S4!=pgQbtx7dcycx>#5J9a_-y=|mooOo@Y3j?@y67$7IAhgvCy0hiZa+HpNj z>W!|AY>lLW^f}odQqDI~;-yxak+c$9G(?Q__JWAe7qQ(!>8Yn&_})7+Kp0tY57t@D z@Z783IK_@j-RRJwcT*iMz5X~LL%$MY~_E8?x;rGbSCbXl55n)I(V`fHK*Zgp|T zU;^)_&nLH60i@ecj90wD=`8PceL}=N@tgNggH;#I@+xupZSjA`Iz2L_26o)rE6YmD zYvb0=&l^A}yQwFKiy8al_K&6fl;zj6A9G*Z$j3wQPxpHk_Fj1PbS?=Z3$kc`3M?gJ z>~3VLCfc#q*JqS70!3)aSwE6AwE!{v(+-|aQ=E2H8JiA`VwdHbiAVXVgxBtvZf*GE z;NFwp91$h*g9*pc6gHJ zW)q#>R``K$s=Os@SD4qMeyJ%~=qPZ8sl#;~x*Rs(6IbUDkp$y~E*vyWxp7fH;rh=j zi9(MrU;h?oKxhs^_D#y$Ii5=SJxs*hTq^q>CS_jCV|tC8VkrX3&*_y#Th_$VcN^Fi zP-wae{=Q$QoYTV_WB4iN=DA{s+{Tw8H02tp0=6kWN2t>!->}V=Pq6yeZa;ANS-97G z$LRgPxBY*+vi%Qa>Ow;}5Ufc-&7s4!pRY1r_N|}nC@`y6=b zZ^eEwvxi^4ZBL02O`Zunjq=C>pA16Qv(d?K& zb_h~48jWnb%TlZJsTx-cTM96Cv(Z{6w`?k&SgxdR7;ZZ8S2TpF37fleB&gMceM+ae z>S|hjdUMUQ8qHRb{?phFyk_;>+V`|6e707~Jx3N0PM;HY$QlAj2xW>45pUWXsU#FV zx2rPp=+4nl6+picOmWiPv&5qA4K;y3@W;u_qF0VCvd5YDbdN=~1u(ipjNz3uss1l; zNyfDeRf&SWLmX9uqQ};l}UbjscyNNd-ESw-s?WT`B=HE}m{o|l(_COZTu=%?ai zF|@Q^jIDjyQ)gidMw(LIbKPzefJ7#JtSHjwM`cqI05aRq>IIO1UagELkV|eOE1M#L ztwTLlxw-s(%a=}z26r~PJG_sw%lglq(wt>u2ssKcdx8Ky9kmWC2j;n|1wBlo#h=;c zqU^3D-BinQx07BZ?y)W0V_V@R{O0f2fi=XfbflFEX}qxEraGY+*@0am+%2l!tj0`g zZXOedzE!p#YSV{hN`>(~)AD5q=W0|XV?UNLm=Q3Q)=qrLXJ}yll~a+cg5&_YS&qBY zVFV_4OWfoZdlzv-Bf`peR8lV&?0`5+Vw>2lcV4-pDL>7Wxz+Tvpu;obd$afE(F~ zgR-B?JbEG@2wrz_HRc+c%}3f<+_=$Y4!vGCi+7L{gzXBNt`M6*<`$Vn`=tPt3_NYR z-)n9gMvftEIUAyzisl$ z;C=*Qhki!6>Bfr>TkjUV!$)f7eLU}*-)&+^%>z% z7<;u{YK9r78GgVjl2xnj$oBai-Gz5;OY5|~f4cWk=>`9IT>l3UNuL=V!5+8$wa8>f zBm(s+#gKw|G?5h(;YOL$PcoY{yf~(THN))08e>dDYziEzjbcBY4STpMdMztY@`MV> zvD$FQFRG!g;b@Zmvo}2JUKcyu&O4|G<0>vPz(k$AP)kJ30 zIKqT@K#s2xPW6Yjj~}oVTN__aS6c{sBd==RyVO_E zjcK@kbH-5TUP7koHNB@`a0bHp8<>FOY%pdK60?7H+Gco$7Nf?VjmFUZmUyHXasSDt zZYFK_C10zfPI-_Hgw`(XoLqH3KRkAj>|kT_Z`RHWs+1zPWOOGV`_i%5(!WSH{6N!O zw=#rQGUM%sPNKhi##Q#Q;$01^anu8iNsrUSnA$VCe{x$Wap479Hu?q|XPRD}MOGAU zXi|HHYt@2dPxj9|^I&Ons=4!|zXC7S39wum{AVyfue|i-aI}Us*qtEvEsGvYhr+AU zuXz}#zFO<3D7#7iAm|NkS2~5w+$bHIx7!hRJN_KSx4x+rcoE{qCj!nVjXT{lLL=|% znXmERy5tLyp|wAR|8c@NMY;EJ{rv1uS9Fs6siXd9`e%~`D-K7w@=RsFuKVJ~zi@xW zaujqJa-1JeoUy+7o4$FonmFkLR!3`xtH$a&cFo81AE>jCx@!tdO?SYn`A+z2Y;*tY z&CcWU(z?W6*4rGN^_CW-uxE1W#6ZerH(0E^{NO>f#wd>IV7-m|WmWyV2A#O|w*HvY zxSV&l^1DogPc`tB80%rj+`y@spYAlc_5|6)xcnNP12XqZwVan9HH}F|KfbG8nX-Md z{5V00^`{sykSb9VasklEPTopOOUwg#&3o~&0sKu^&(|1#H)m&8+udUOrOSk z$R?n9?pskUx|o4XvULJ?dI3IvdsNhiNEn+yR6I%YFD9_#_~Mcr_@)bC zzM@&NFKb?0AWb6x!Kq?|6*{7c4|g|0GM1&C{cTzd{lwXK=@JS%`jF2|)MtZN?;UWs zOP}~ODV)FKJxrsk{;3f1PjLH0da%5bK*Ck?16f6W&U<~@NEFc0N`*UUMi6gcXQL{L zY1<~=*SVK{H}2uBlC?_sv-mgq%Ub)LErO{RnO};ag<|`p$eZ z$5e$vMY?ivqq=>j)6@`eC&FzMz&ZOTix6{)sCs#Nuduxi1ob|r!_(d02h&Y!F%Fd) zmjp9BdZ)&gF~RK@$cpp~00+IiO9ZFQm^Ur!k{no)k`*KNgWHMQAHKRyrIR5Skx z0QhhD(v^Sz44~>*P(_ow!5P=tO_S&QswXEN^|LM4i7~t1dnMe~%)|Cub#8VdjuVUq z$aoab_mxhXs9oEeuM>a&^4{~3mV^vX)CU3rAZN2cdu@{s)WLa;!p@+NinQaHctxp5-}13<(=&jpVVnSuXq%dNIihPHcHLdve>B zv~)rax}*VM@k_@JX9Fr7#MOL@D2P)Ediu&MuHa9=9R%IE6M2ZFs?d=ASY%4vJyhas ztZ|HZFjxDsrIUnpZV_4Rl~0h2?K-0v1tNA_fX-`6oU%D1ZCN0uO(Sj=F0Mn)D6y&? zCj9M4s@DVy>|+PV{KDdW63-90%2Yq{@j+C%umESEBqFk9j`7@W;3QQ}s%Si9jr-H? zZx0Uv8vJ%>I>ZKM<8O%i9O}a=={6$RjH5V!$01clt~35D)vt^#o8oR-;?2`PvMS`= zD0@^m930cfLI6NN#|42TDd#_PAand)2eavAY+N3O`mvR-F)x+QwG@w+l;7lwe%zUq zE?(b7Pec>d^dr$*W9OC#S+qRx%HmP@qGkyEkPc3G-Kaa~)M<=}EVpNVvljk%*zwBa zC813F`-Y^lrP5^kT`p#1`U8kE5kh?0&RGcJ@S7IY1cRFR%z3w+Wz^noF)T#dzppMo7D<&`^cHep{Dj)QMhooL9+TlU5cNF4U`%W!O~ueZ;_y zY3-2*2mHh^2mSeY!h0mRw{g=Upastd2}ltS;|L7+{7xkV&$65UPpI&JIX?e?r z7yru@^hHmF5j_;xzB8~{)}B^kFaGzhDoEBV!1j~OQIWRN@mI^EWXU33(@=(ZrFPE7 zrJ!jJHvG9JRb@UMrQAsOYtm6EK63V`QeSS^hD~(*>RIuUBH@~^DLyoy!Jo|9gc!+FsfGs^OW7`mI5IMYv`2t{Rk5!*gWeq-cz5kNMOl2 zh3HRr1gPN($N~rb0DhRHg$GH1$tKt-Y4*p#fpDb&3O~&Xe0tWjiL%uwOBJwQzJ27R zH=USgOv)YTOGW2jH~^reaaW>ZfH06k8l63%O5O*9rr_qzKTMx6fSs$*k2-KvD6RM& zwDSJjw(L`@InQEt3{eu%!;3?lh46#jPYv)NuI@yk>Vv9PV?4Aa;Yv+V8<`IA{72u^thFn@lPt&`Z-p3~K(`F3JIn28Q1hYXrHKmP zrI%eMN||&3~x-d?Q>w)$Bld6>5T+ zAeyt_%YUAc)kH1+(i;z~H3|{Nk7yueD68O zrFp+DaK={!h@QVKKGIgu47mcjX9I<%jsIvt=nkU)*?|bW+qLyVR2oR4h|j_gh~Sv< zm!6xY`-4Zdv3t~hiM*J%8DK$iiOrikWS={V(2nZw`_o5M%MV3tOoKJx&=1YkQNu{s zIMrn(X=u&+_IBbR_`~7a8la7k8-hH&F~9&SL3P?q0QGXYpWe`#lMyxQjc$S zQgAB!R9*oB<;`;-=#qpkM03(=VZY=-qu#>CXS5x{GUl~aaW-X~5i62=`~)6kIqHl` zAAJN^AK=VVJp>3+q8Qk?X3mIJcWg2wAk>e3(C=bCJ^*7Den;8efj3vv*`OjXT-mW% zl(e+@040d44kcbt!Zd12`WtS#lZc%vSLSFeDf8oQlE0<#3G|ZYuw;rnrGw!=;foMM z4X_i&v0uM_wF7C^#iW@)`98eT{NIoeF?=tZA5%f}*_(SG=cAT#WnE&vL$xlGtR-I3 zd=-n1GlI8n{5+Fd!SIcjqOD#qHXEqlqVnI0QYS5wKiE}mu~6+mkz1k;^U%aF!HTmI z#9}#6P5gu=hA43UMusD7UN!~_H_DbJIDh{slf$uZVF{r%Rrc4nq;9NyPiLsg;ZgQE zSDlqa57JD^S!R~3P?XR z;m5GMox8rq@r=e4*{)aLM(IBf2?^8VK2z*?E7KVxsLdL+}9;SO|<|A{DhX9lID;?y_t(f@JV+7NR)pjJMdBP{7evtc_qy)krG*~ahQ{%-2z3)H%)?zM9VGt7b($BpW*v`he z#3{Rq5!~>vH?Meop5$Hla07PA=eKnF5{a*?Qjk(T{Oi6?z@ZTq=rn7G*{+01XC?4P zDbvKgWwPes4webgM9H2Zr2+BquSE3<(9)l1WC)ZuJFD4z%{%0GEg&Jufox1mjTncK z(#A*30&VY&ecuIG@O2#E?4tHxj#v9>+5r;pPwCWcWIIFtpV$CT2N{!Ppntq}@}BzQ zbp87T);Hu?lu!Q^QvV%V|DUc*C-kRM<5M7#llk4Io_ZL{7?%*x_S{} z001M>pfM0{E-Q|dR7~r(dkmOPpi?Crt|Y#Cz5eJc8z*@mlzaD9U#PqQUJ!tIL-*mR z$?4hecZ&FUV>4*#JqvnBGiaPCR;dHtO!*Df_P%Lf!IhNH5j8uUpbMzOe+fGxG5oFZ zlz8Iu%!*k>(w0_9i(Esd@%Iz4E^dz!71*DAKzPYHW{=)7pEO;4H@@k$W;&t_ac$zk znucIt6xHF9obo(*uP$?_UHI_mY*dNB>l2UAH3iRustlNnZ*NO{IM z#pN7|QGI=XQi&SzVTf7q{~oHFWr2OG8Re&*>!qvauiJhXXE2^~i_NO%Cu1|jDru8s zjab@SNQX}zY}>qhocaUmtF%5l7q5^k%S+%n=a4K8Lk{ep zVsNIt&@@Nu-4qq#F~omi>Zn7jA9#nNAa?APCD}}&zuQx`dDXUU#8nL3rW$Ez@7;GX zt^a3Q_>a_ZʞhCsn3K8%>>8KDj9ezJ<S4QVJ!h*CcscZ+9bz08aj`n=yJU>>LCv+uck><^)2T$8CNrf2Ya*UrypEBm0P~y zM)o3HOA!*6Q?ROt`ek$ya>#!#yzBpB?@j!nZu`FRk!2`MWZy=JvSf+uOUa&4*_Wwo zArWCPj3xU{h{}>g#E@O~on(ouWhc8C%NUF~pQH0Sulv64=XXE}d`Gbl1=6)I z;ABH&$j~L-7o>GOwsqq-gGW&K=J||?0r_FPKeg>DAz!0cj@>~G zawtHCxI+3dyKCu{eY^->-aC_q@F3Ou5&Y32sRVrhR;g&6#Zy&)HK4IJ{;yKtf5a=( z-|3w&IP%@e0#(De(shYXrpYZy(?6B83*(K|jCr3k!`d;CJi;RlZtuRbG#2*ptjlF7 z#zdsDn};MCM{(2Sz=jn7tU_)!ChIJZDpm|1iqCTq&9{rTvZG;I_JQ37wiMvXqb*)^ zJ3dPQ@X<5i;#rj0hi_e4pwnwd(^M(`;+QWfozb4#Gu;vX@?Z`xw}YSstt{IG^fB8( zHjzI4k7p39677h2A&aJ+8{PKuCtR%(*((2Rk^E45G+Dx2okd=L4))tdwS?5yB%p7z z`B-5fK`_1HwRdCM{nw5U=50liMtFgqts{BSX}pqUsf#OS&95m}W~-{GQ(E7>^Vc~R zil<78!L67#RkM+SB4b{bC7K%@9GM257ro7%+DIepjd>vokAhS}2=`88U7EOYV;LpL z1d|cE(rn(kctvQ^j1E&sk_doY-zAOU5{(+tRk12juf4{O7ZGVZ&w z-3?<-P;P}MIAbxvS&BI2noa}ymZ@K-osC;Ry;Isf75fr6%b?*x&|2l@Zyl{1M)TMx3Ca=X7cOPlIqO8g|;2w{!1)Qo7K;rzbu@5Ey!DwbEJg?j29O?5}E zx6|cOgC)KkCs;h+9G-YR#^=)en(6KHq;PIWFi6e^7`EaK#Kd98aZ~fWvct4_VshwfmOdybf?)~ z36zAD*$x}N(M^Ht+Q5i0y!Gwjm%NKttyv)=Tx4%jMs1;M3YW=O&~d62W9198oYpQg z+&@Ga+-_ENnD(!gx18H$((JjYWpzHl|Cxo`X7R#@QL*v6W0h>1F$wZp%c)B=KZ{@OZzt^PhHDvHr zN=fN*SK@G+yw??CdVYO|2N|qrzLKE>>Sq3g5BdQ{I0LhWAr&q{jPVG!ZDVCcvYB=I zMKgMOk-hJ397v!XY_6nHyyGo8(U9m75S#=11j09H2Uz?eMlt?$X}UQ2X{G-BE3x@< z2fKD#Pxy>=({8M|!Rv9+&)D4(lxL$NlX4p4c19EULrpZ#wZ3 zVj4{vhC_D(FU3jiX*AbVk3I>G`T9ka582x*Z!938(gAk{ZZDzuZk%^bp3P$pla#`r z<`#n&e(K?UB)>>Jp)LE(wVy-J|Ihj`VSoR#KC7IF7w$qW;{^Y!`Mgq{X$|+G7kopk z!bmGvDutw zKIq2EtN`B}m=u7U7~`4)?zLwV7(lkqj;tk{C6u=kV!;Zxpp5wUj$Oq}Y;;a-7h_y9H9vl|vYEVunHsXvwzE(1 z!_aoTIq&y;A?R8I;^ZnJdZWWHtSj`B6J3G9l{E3{UOrt}uEbW`hphs3JONPM8tfwe z@pvVI!^r{@Srs6%X>uWdNRLLqS0IT^3Ny$NRL7Ndhe~olD;BxfJ#+bol4;S;Zy0AI z`C2a;%J#14oHKHUU_^d?qB#FY9|6W{DK90?FGIpHv=s0pkb8306EXXUMTE zKA6B?NK_yDoINqbjSWH8@F5vP2&FhRxY5Nt`f`k_Kp~ zKWQc!Ljt1-7uug8_1Y9{!YFp!(K*T!~ihu;jr!@iH}KBZKW1v}&d*dduGcY)R`ts87e+JNt-_ZE`INq_V&1Eug`SitouW$Ja01<@& zF3~4-+)^;P0V?1G1Lpg{qK`Ja8C{35{&7otP3{n_#=!2=9lsX9oN?2%F1kDqNbwJsnpz zb=0I39hxLSSFOEcc3;W`PT&?N3_;f%KHBr-f3TkD?=QD%!+sGmWY_+4pK-6Ne#XQk zle|W%0_ZIze)sw=r^&PXvly{MPH8aOr|}KOpEgLG<4>wyuesTdti){YTx0cOgo>Wy z(eA@#Wngk2ldoVx;1w(7bF*C2goqA{EFd}a$mBY9qN3^0{<2THMhjWO?WJSe?EJ@_ zDAf~v1{zuno`z84y;zB4;WU7haJK%pp_H2g&=Iyzt3jx#hOI*7& ztvp>#8K(s(SBx1-2J7iG{@uK(;(XfHR~01^vcVzw-7E9nmd~JCI4Pt zz*#~#l6q$ZexUTQ^;%=4S=%n0N1_X9cdWYj<63NghAi3PJ0bqce?S||Sa>bsCC zkRF-K@#D0C#WZ=cmAP_@QQ%pfG22}!|CFDZ%Sx&^SZDq9a+ku%jpLZXr}ICA>U{a# z4Eyd-^YaC%>a70JIx2-Ob~P6wZRf#|(v`!FkHi$kpU!@z$u2f&oBWOcLKC%$1n%;H zlGdlazaRzxhZF}1%pC8Tkp=@64gZBW%^kHL`)MPkf=+oSPm%?SYnMV;u0tuZJO^`% z(dFH-GF$gLrpDrmoPA`g`TA#sfA-kfB`OH2^Qc{{=6MC#L7c;aPjFMkr5`}JHuXXc$jqQ@MzgR=xbJTh6s50 z0v#B1rEII1)`Ch!!G;2To|9Vu?V0apZhE2)68(LBeV=^ndw65dYu)zTb8ExE2_L!_ z1cOYiMg`;jdrI=77raw>WZgfVE$7eOm>B;UI1wl@UOCIY<|p|2FoFow%T|jA$^`nA-mNQamK>gL$q2LjDA%kh)Zb^YL-*kDZ13(JK9qm#YQZIRxjv z=v@|gyOgZPO(oZh7mi-*_uj2PUB0OW#PNAH;GZ#30H0^gYBp>?qMm)RbUF3Xm1(5e zgx&MGz47?e5hCB|n}b1Geo$|n61dU8_lc>5>H+aYBv6l1JJiqq`oXkmse>U zW>}*4a!b-;nxnnd?oE(+a!0Wra64sJnZ_w;Ks!a^1V>C8D$B~l#RWy2EUsEJzpDN& z%XPfpi9D@|Lm9bxGGPfZri>l)qpy3}&d_U7%$))~-z21`PQ(o?K0R-$A=z>bUv8nf zd)_=v`YcTgxg1Ywu9*ti=(Y@o9O-kvNqF3+-nFW~QR}6Tq9cYrcql;KVTe^K)V!^{O*GLWL4{J%2Hdg-u#-U&M>+)1KOJ zQBj%0rB`G_rWHiB(*mzw;+<1j+4Eccw@>Kae0$1qB;aeEnCfuE1(BkMDgpNZx)Vc} zsfg&xY4P)qkIjLFnHkr_`zYZwIa=AC8?F5OfX!cW8AErP+av=#w?_AW@!uRNDfJSG z4E!Nl6CA6@;9yu()RHFpekl<`9-5ur4H^waM9@7_O8_W`*+0Q+3lcziqO;189!a_Y zG_+(YtbPe34K80fiq5kyQ+(r%32UBb!of}U>~oCk4N$b;fg#m|<289jxVLXuVYQ%d_F{QL7iYWU z^jd6$W58y~a8&MiCR%gBrdaB2yYPAbD~okP*hK;IRRx^E)OdSlL1YvAbZMN^rJrf0 zYhKdNVr$a)lXS>GwmxgYaE0PO;i3oGYlojhUr5YEarN}qEuFEGliLa1mpK>@t148| zns2;jUKy`F*IX07%y|2$;W{;x`v_<*VF(&Z>fi}+oBjoXEP-m#Wg}r?r`@6{3{;C7 zrk~RW6i;5bc)fDEg~~(V+C~T&qVs9)0kB72-)JXsfB$sJ@_g|#ncXj}-iEWh zT2>J1rQbp$TsS|UE%SxcQL$;G;YxE+^#pldox3+Pe!H+!LwRz5VF&Dm~Z~`-I0}zQj%g@zg{NYe6i@s(DE(NS8pE90C331 zRG-+{iqY@1eYt(lH0!VCz~WWy(9|1avkXE%cAkGJ);s5~6}D6x+nxSM0`la-(D=l@ z=~pfaF$_*bVuJ&|(mdbnMKL!g2SeESbO?3|Sq?mk_wC|K}%(eLRD{(Ky zE?UL@rtpllbL8KcAE4p(1uX6a5WO*xs;kfGh;P0;@D@M-=WE1rR*iSeK_bf+be)*VTV|ud}^`ZXOVI(ieW2j4j?8nrN23}>(_lLOYRg?pYE~~;rmifL6+1W8z?iU-j;RI0rFU(RHp^h zwjjK0!lq!00f+4+-t8JgmD855$Ahd5#O7={@>qvl_BUzGu6K3EDX4K;nRycmu99dDbtjGJBi*=X&84jv!TRssa(azr(-N{ zTxu{Kin?|8ye?Owrk2f(-_2D||C?j-4=v|!)(rr>;r#J&V@l$SF{kI+Gw)(K|?G=v@Op-j^^PfaA0nm3H;iudZ$mC-wj=!O zM+&a<2Dd%8_y4)BJbd^=}&ZMw%e)=(+ZKjae?HE#&4(F+LqU0Bcq@Ndr4=Ps+c6^+r`zCvMnx8k^1 zbik`jBdIF9rMX#k^(9nzS<8F&n7My`3bH$BS$ku*DC9QpvvsA!Wk&vc zC7QvB!((#&f>@ExgJglEz9ouB+!InubMf@_H{_<&R0`fT1rknhvqnI<04XgFTT4s% z`AI~kW4y2bm0LXs-DH5OzmK1TOU#|j4M8gc$85NM zC~0Bq1%_R1wk?6a`q&(yTqau?@7}pK8x3ro@17Pb8IF%b^@pFEZT814eOeRg zxuEp)B52748A2yzusd7wz@G~ilUbclpX&R}zL5HTh$S*p?Pf>hjVXHi`&j=+jjD4A zTQtSU!S3rM9vn-?!}ELb-HsPtD1)a;_2woChP5b0(nZl2>hO z`hJ?CBwvK|oPYAM#6lT#WOr>exI$se($6l7Dl!_r;hDfZ{)A!Pa3{F6TYRYU;!19P z`1DYm3;DD4h8+8%k|9+IGcI`3}~76&6ZW7xs#=Wm}qTtSl*)!;frzacLe zBk>ZBDmCv0KISM{D4thM{IgwRa2DZS@XzrU$F`=6Q22xehD6p&Vm?%PiwpUV4!MR> z%f6j`Nb^}w+Fe$Eg}Q~-@1i%?T;jD>erU^Ud=W1F#`}KzqOoeb{34RP;pEUz z#C_5L;7*CF2sm?OrIxIXjZMW>BAlZwCEA2yg!6SP7};2$yoonfCHMS6t%*d5U}d8) zq>F?A+hMl_Gx3MD$vSh7lNh+JG4_MH=tuJtHyW38lFWCn64g6(4Q0UOUEGa5@l%XI z><#jy#6{@DvFW%pPzAV*t%*dhQvUne&B>;GuD3%^6vvX%i`AEAEvJ)2eFxgiUqfdp z9KJYTh7@im_PPnve|it=d;cy-RoW}z3^GsW$JHlS_r*)hx^(2Yk`Ny%4XI#3(qG?l zw#;yU-$^yu?$)oa%DBHx_WhDc>$Vh~+y7&B9%v8LI!Rj13;-9X3#wGjD)qV9BhG!* zV^S#iNG!k_r)N%JLpgy$l*Ze&IrXC)42N=v(mBeSjI%@g8 z2x|IkC(36a+a)H{DdZq~A!_St-SOxrByhFtlQ$4cn+eMhf7MMeniRO!4&VNXUb|Jb zE;CRQg_3`d6jb)+{3?f%r>n1&Y7IUgODmiMzfH-lVEPu_u0vG(3zGc}WXF)HAmyxP z5t01@&>2r4Dlbmcls3pC9nM$xK1%Y|&U*22)1L#o!{yQpvBN$6C|FStfVt-sRhKpF zu)CB(h`c2EUyfKma_gu)`2*UR`8 z5Ty&+2ak?N(OA0T;Rd=z$5~D;#@2g_%OkNMG79aRYWoi0FLx|rn_fEkaaH)>q zxB;Lf@4t_av(Az?aUpC5y`Ig-pcNdYRMVA)OLcnNih}_A++;jdg%hdO>Qga;bi5Vb zw{Ewag^*lwts5A!jwh2FkbFbC&qRik-MR!2JYPCh52nOVFL3+54QRzxm=ILqQMnP_ zcs>75-PDvN9bt>NmqcDo1VWBa#@Xk$Jn_Ez-mRO@)5BNqgkQOOcdjcg>l~FwEp!yL zEp-4uK*O4o)Q}5;9;Ne$tydEBqP^V%YI}JKo);Wm{{mwkVQ?-a-RxNOrlb!TbZ0i} zibUYt+Rzk)8go}m#L&a~QEnF54EtohdB3OBEt2H1AXf(Xk3cVTboUZ`2eXG6Z5=Wt z%RzBs>4S@>)-w?i52BAJi9L3{`HB4WvCOYx<8u~6L*$uP{~OJIw>j!j3ec*P z`H1Z%=^6m-A`G<~bgY3nc180qpIx4VTcc87Am8};VV zFGw`l>I#n*^W;iC6jf|hPHO$dAXh?fy!Z(e0gVD7U=90oQ8lE7)~A!{iCqN zeV`g{QvBftPH;-GZgn zCU?Zo!vrwy2B&X}hc@1hHki);B5y6md<$@83FQ(V>ZL#Xd2!jSW!*rH(sz(3jcIQ_ zI{CAg3dyG~mpZoC$PJ-<@^uQ4PxrPMZjr1CzgBBcS1&8Y9`IC+82&3w2Gnc~rrwqK z85+T?ThVNzU%r$9>&vztHl>XhP{F2u&9ytfyXn;$NR#67P23wrlDOz@yxy5u^mAxuE^vXgz zYy2mZLyP!JIy@7)=1@6P$+9K?rQqllCH~UD!6f8s{Qz;`t+$+U`@*lNelYTAyK^Wq zp35H^ss_IeOfcy{Prp=$7-n9I=x$ORQlfd^S%{3@>kW6DWaQiNH3%q@mxBu6F{)r# zlKKafs_K_OBCCQtM^qXRlh9x}$Lm3xt^+g3Kh}Kv6Uu1*mrv_pY5vxyDT=ad`l7WH z7vNI0%F1Umk4mi_@gB6@CoG%9?(LX2f$)YLdLO(fhz7-l;0*I1PPypQwybOrF&z%fG0j@^&1Vojdb^sVShymr8%$6!PH;}{gF9=hEEor6z01i?o zk&0~`u+Znf%dul`nd2%GR^&_^9Nq5r>!?NnnZs-Le?hEfEjCnO;CC>^qEExN{@_u5 z`W?1rM;w#Ny^5T(xI!#93ibq(gwl97{i;XZ>y&l61(H27)jL^_wj2DBb~C8m4W-<*foN65s2K zEc$ll7FRDv_6`$om~w@UwG#-m2&V`RfxhW3+87T*F5pdyU^^IO0=6_1=%pm2I;j<` z!7ASuXH3CQOKY=MiG&s6YHG7N&E!jT+Dbw}0zTo<5 zg+<67R{Z)&u$dTC%;4&lE-zj1a4(vJ<(vspxpA7&yi|*{Jv{=~-mRBAQ*Hn&1v+js zH2?#OEZ#DJf`LIqsJKG|b}WiAb@bzM<(C-%u$p7F4c>@#Bv%$0RFQLTrhbIuu2enm z%D871&jsVvwu__bp|^c_K-PFiVF=n@b?h~VK4RR5oj|9-i>Si9-lucJBdr}erktcS z+=UFT>baMxO z5=OwioC01Q{#-4Hy>!o1vEQ>{s=s&t0>C@SR3p5-Nqh*UKW~I_IvnbKU-{&_&c(C* z5Y$&*)#t&Z91!9Fx5!kpo%oTcJ9<18>?c!KpHHL#gzMXE&9|?}us!2zmVXmmBTQQCQsj}uG07(S)*}% z5PezF?HFSa1jE4MPi(9xoL=r+kMhBg}Tq?dG)GxMn^WD5j?@+5<}!Qk`^5xN`Y*1I0j)vSi~bZ#-nBE4771)CZ}g~M_xJBNHKd0 zRI^De3g%%QH|tK57A)zsOmSytET(z{cOTS9R(yVg={K9RqaP*fBu&lk-jhb3P-y)@ zmtnF+c3|<@l(~f0{srqMQyC*wijPSH1Go%+pRp1SO22beek^BWdo3bP>8i5{ z6thHY2*(aF<2r~l2>6N5B~&*e*Dt*-fNy7>%jQh{!m{Yx?RNL>tCxNve7UKAb}Sai zVL)&S*BeEzJ^{hIMG$mGS0R50S2F)ixOxP%+$f2Bq4Qy-IlInOz+Jvk__OZE@-tVx zgX)xaJv?p9OGv$Kt`_V1oE^n31z)zq*|%a7z##8=*~2$nd-1PwVAKRrPn=5|Iq6** zV287n4&rO&2w(kF*Z#;r-oPLTPA4)#VJQ2DrOx$)W6FDufg7my3GjWvQgr z;40>%(dUU;EH$>p0Dod5{YWHORMXf!@9h0E5`MtthznCdk^}Lv6ea%`#1D*jbnFV$ zPXw)_7WW!VX(HTb7<vfDY);WqHP3d^y^ z#)g-A&#wQ9ewLVPT1^Hf2%3MuuzQR6RIt0JLG0gPB}M7(37?oQ9ByK>w*DOi+F#xsOA(@|_sajw-r_hJMG zRCnqj2@68RjGa($5w5_0%lz44Syu@9`7VgFt_=)y{aDM%cdso&e588n+9kH>S9%sP z%nJx?uj%Z*tR;Hx92)nW%ni>5oaU!VKhgMwrXq|KfgSr{cHTR+Py@fxJSqd#tbHR0 z*qhbDrFSG7KYI=Y6ay7Hu%3VaEj3+eCMM5a44)w(ffMi@2p6@J`M|r3S=o6)!uTHC}8(3 z3S7Syy-hlYB1-_ZPxL@zp$oZDeYjLwFO%SPhy~YyY_!`)a8jL3J}$8$`y+97)z_CEgewQo0L!D0jeT;w4OmC! z|I+C0M$aejfXns?peX$vhc?#GWCV-B)da3Lypd?D5`E7&{p2R1F3#=X1esqt%nvez>8K@l?Z`o@ZpS*`?!eo9+ zfajm(;X5N*Bl(oG&Ad!*Cn388;=j|XdL&>2;$hhnIFO{CszAx+St9-0j{_4PHgBd$ z2r#(GyB0}E{qbl1^OK3h(+{BPt0iQguBz^5DV&mH$O!Ex%0ls|hq95erZ z^k`mQ6Am-6Vn}!i(E{Dedq9Eq2V(a&i31I!rH7+GO$7FiT-v1bUnqQEWlD8F`dey- zXAebrYquXB7Sl}gq#lhMZvZfq<9~+X>CD&s-PNu#>Gwc=@F#m@S9I z5C)#!K=4BFBC41;RzedDmVknL}cpP4M(2L!S^&U4}?BL}<+Xc%=W#_%z~Q)!-pWNb-!9&wBZncYo`qs%5E;Wst}7B%Fw z+g-@5YV^79{ioMMc^)^c;ys}9fB4hj3R3-#D~KE<+IG!Hd{b_xC@b`8yG^%MO}i@8 zf&gnBv!HwV`F5gxme%@qw`f4Pzv;by>9?Wooq$TQ_x_0LCXAI7pVc&m;RHWc6CRcM zHvXe)5x1gIP0Z%NAVU%luX#|HCbh%U0Mbh`Z14sm7hRD~OmaJi9{&P%;5g<2_=R+v zK;NPoq<;`gfQiO@Xkee8HlQKc-Wl3oN!ohyZg=>-yu(tm!D#Yha_9dCDecVv{qM2= zbeNcrd`i~Gk5@3BAJ-41tY@w2lFi@Ci=uP5AzTm$m+AVYK%b3lsFHuabIo=2cE7OI z;J%m`ISaFV?hG^f8^lvJCBri)iXWKD?3gTe1>jHzre_V&W9|`Ad_H7J5$@<$&_d>^ zzzL43)4@*i58u2*CM+y(4c*92I#+5Iyl#NbHz5dN^SZLJ6_w|{o>CnG4S_nn-h#%c zclw1bTNLqT`;fq(3)XI1TG)|Ji|!7xlGRN8t^#{6H!j=FNIT>t6}1<6EFKt*+>+|0 z82(0)0E(XH&jq9aSC?UEd@P$ArT?jUb9nBgtNHa7M{QL+Nu#ZDeQbM^UXjJL#%Eywo2f{}lM(-F8jm_xHheb(6 z&Oe^e>hAZLB##YxHUJbCED@%GUu`b_NSPOpzmL%U`o8V^L_|9^TPHtv-BvAme8wPG zCKf?3wgDAyt~$s$TV#PY$}s}(MR>MA2OI)o#Jq|hKJ+T>7`ALzF39vU+Aezn6xcszsDRQlfq{|6i5Xb|_+toa8?l#qKM z^6@EMb*}Pz&OwHDuGqq81qgISM<`i8FxIp!n~lVX97Wy6QGSaL^)CbrF+G2}&X@Ya zOi7a=$xzyBA+Q4X8@xQm^Cch*tB-HWi1Dpk>TUR{Ye+j%8mJ#oFj6a>*H+>eE1h+OJrWmv`dxN;qy=~D{I>L6`v*YJ>8;D zXLGh^uLE`ST4r1gQ5eGm%BI{B5YqSowFTloxam}~ZZMB#pXl`wNb~b(mwA83qpTt- zW@}JQ_e%Her2`>Vap&W>m6F1rv+JgeRxu(Mf$J?T(k2OF^U(ro&E9W6r9%Chez+_UP+D4)LeQ*~T5$|74uf4k&~kS{Qw4y4Y(8P{1DU%81%f6bUrgm^35>MwwXhCnT5 zKe=%_-E(Q88?St=8Fy8D^Qzb*Cz`Vy+LJfrC>Djj2dPT!TowFMpSiFOl*UY@bVPwF zS5)sK1A&wQmr1mptHtr)!q$guS5E%T4PnP4|^fy=F%Zq8Gg&|Y*50h zy_5;Ebv0fd&l61$&ay#$dtM9eC4ie8aNQbCV9=BdGny_9*a#Xv(EG)DuX^-(@kduT z*Ec#$!*8TOdq7!%1K*qg`V;o59j7jLN1iJ->V7qR`Lk2(rR!w~uBuG#VSH+b6C4D> zkIp7zw(T5mF>mH+53g@=J&WJ$*Mb;AYeeTANebq%nRzn=vho|u=F1X zS;s3lLd!5OX}$%doR|=x9XvwAsB2z1<|;W;W@+qmK`eCpmS_7sK32_;hjky+=Z7T|%#4f%;^#w8!_LEiT z8OCZsy^>%8;t}LPbpm73WWcq4eDjyy>1)BPLw&b51(}~Ub6YmuJD!cl?z5c;HkqEL z1acv>+ncWbkZDBUeyv}5a>4wa^%Wff@8?g7vggU4W@>anwPwBq4B-=4MI>kQN7bUkH;SY^}OujIT zFd}tKmF%Zy@qJ&aPugF76px&Q6L|=8D6crHN3GpILY6_!-uV` zU-`Jet)j0YQmM@sKF9nr^^u%PIfDcsQqiecI1o2z4hp^^UThb#}JryeIp-3_Vl@IA|+|XU*^1aw2vn z*Y9FtVQocNezei3R2et5GFIy*Sx;&D++dp<2Q12u@)thTzrcLLy3=3!uv}W3Q5HaB zJ-Tzq_aTX7hsL0Vzym*d08R*^E%M%-`s813Rcu>VQ5B&Af>+FPmECo(J{x^Dl!T!B z(~ho!g&qL6WEnlzGi07V_I~Ll5LKnZ@}Bp++p}_V+hpBx(8XDTai9{o(X>Er((-4W zA&1SAs)!%Zc5%*|ZuXBWYx}EpCmxbZdq_@ab-v_!J`kXfswlMn>CBokgD?;{yay5v zyC)UpOUW*quu07PPNcMWM3YF2Wfco6$M;ty^R<_(CE~xleU?g5!;PiszD-;%B=X_5 zQupyC^8t5l&%NB^4SvhX{Ud_He+YWxMAZ547~Q4JF)*;MUE)TZD18TAPBjs-r|_4c z;K|FrhwvW<>Ux1TZBeC?VUzU^1dJJ=EycFDI&XPIgP$A--15q7Z*7 zhb})y@*v5*-+yiy^ibZvFPQ*kY|MO9^9RyklK^%*kIAIIv|P>7MJm%=C}TNeO2;}zv99-#uRzvH;|N~q2d1P+NF{@=Kjxv23;#E&KvO&R zdkID<-+YN+Qb_ct9c0I*eeRFF|FX&f%H7IN;XhtZ`U^ zeQfK?-pzR*u#kbC_wmesrj^atNPN!Y$Fp$<#otWUEsN9Ylg{<{p8aI31{6Udl!32I zTX>WSHx^&3vaR>RBHVUz{nOJF|KyUt%%j+N{dX0jAuCOc;9JOcWYZhuTkE-srVmZ` z^aK*!_=>M^Q}gc2Y&2JYCM~l;_ zCo_u1;sQ@G{L3$4_3BtnU#Q$7FZ{}x`yn(G9@LnU2G51J9tM*l>Z6Ezb+U(@Vo#sS z2JV`D)y~)RTDnYepuCroot-{!kYE<1#9!jM@p$prbfigw;JCov?C@5kZ49S=y{h0z zT^jYQB6gtj{9@V>DFRA?ON7O|`n|D%Fei0bG@gEoh}!=W^YPNi7F|O%wyq~C``c{I z5O>bv2TlJs;9-FyR@~jZe|ZBzpayVHgnnR86U8|3(!+&wKta@8IRYPgGqF`d^c8XC&a z(2AEYM8TL+2qQP*sb_RJj%kIj2#~PeOk@X`Epc3o6_&p0!{aQ@_!*I?lFp@?d%P*a z=0m*qT9$Q$)<)hCFhuF>4#W`&+|Vu3rJ?xo@EbTz&5XQf=hk#%F5atprFR7|Z|gji zyGu~JewmPyW-~&g7qo>F!DG2LTyijVq(g>!r?(>Q->BV}l7Mn(@&*M}${*i*)iKbD zO`UP^PCs|TyjtYCLjU=wgGrwHsojIwo7D1aA#|uOYH;_!egNe^NQiv4j<>lO&C*al zdAT<1CY7lGrLbaLLFS>tkvN_=+F(nSS%10t<-=g|xvgL6a)TJYrb_OAwTUoNr#VD) z0CumXw{T5DOYHPl-(<(GM>%8sG#<<<&hRL2jJUL*UD@8J%*VgNb#0bAmsfm_eZg}- zPiV+4_<%1+Nsq*#w*UtZ-wb4(9Hzs~73huB@Ge3Kwf4DYUukdEwz3_{cygY;+;wt%?-#@*IvzPOv|HhD+_Q09ZkI}47 zO)6v0n_=hlMCcS2^dvstyyV1jU3freEm@8`Tybf9{NU&dE|49-`xhxPW#i&8&s(jv z6)jAJ17DoiD&z}N=5KGVY>vwv7tJE~!v>t)!oT|1 zxwUzBly8(E!VG&=J7&)cNH!##7C)Ib(vf5QC=}mv#-I$a;fwxjO+Yv0&-=RxQ{oHP zbFmw3*K0VWC96*NNw++DWJS$5qo!wrqYwO!zTzjtwXin9e41NaK3k0;$*huzQh+~_ z0B{VbTVRs00V9&@9y{W_2{8dU*49H-Kj2h|wpX28PF7(-yp(Wl1apiakFu_tfATe^Yw^z2||Sk;eZbPW#U(hGhb_!eCt1MJyur;t)!@ z8ozRx#i<}o@?r{FjchG8uxeQ{@O$?83E76H`9xot{auSeTi2)C zx7}lm(PEx0KYdj#PGmb*cL{oy@s}KvHXV(P*#%xrL(URAQgwU!TEd|ee~`pKRL`g= zB42l^o`13rV?5_l>~i20uf1TF$$H;$ko(42O_kIG|7rfV@qX&sM&c)jLz~7nO@0Y5@_$u=g1j z)A{TM=U0)n3d&n|JLB7j&O(k@K;NA3Voy=ar{EpSB}4O%XO*RVF~+BFNHT<&t}Myd zH*vz#1}iNi>e7;tvHl7nP(x1yfIIQVGK_NIcP2)n=6C&uGMXZk`JXP}@Sl;Zu4G?1RUy5hSww z0L2sib^$=yMBPd3+Lh4edU03VxHjt5DTE?hjV>@?2s))&A>MiF* z;Dd9VXPOZ!2VNF}^>q7N%NqWdUN8uk&s|-5r0M79H~#ex<^YV((0GQrKjSkaB+wb$ z`fr`Vd=66Zdt~^OVu1hc&!Z3Gh#m3k%&Ep@nraW+-Sr=!+b~%1)3Y2f5<~+iB#hew zJ&zfMv3&#`^3I7Nj$Pw>UhMm%m%mmVgM$52Mws^yM6tmT^!^>rzaW&}*+grAo(Iqz z0f!S(6UguO6$rXO@1=@98VW8iXqAV>&X)3A zqqu4P=a3PTQAX$_zW9D@=S6PaG zLN6NZ7c2u9sb#no)_*St?-(%)1WE)I!(ZVCt!zul#t7NCymnQXjGXi@eQ`JOB?HO1 z&lI`R>=%Ap@X$x|k+49fdiSl``p15v3-;(;lp`QfIlt%4e&ACJGSqPS)0{(*V ziAMrZ4mViLcT+)1A&2;Utaa5UR;yscN3v_#1o^R)S;@vbW*FqvSw0FOZF%thdgQ&(ej!`Mr2&x%+|1(|bshC|{r25zuZn5KO_ zF-A&W^3oSUBpFB2^F>neH1uXJ1k$5>KM-$%M;n7V6$)vx16r2oux9p#YdB8B+OqE^ zyWfwQ_b6B+i@u9{TYqX@phwMnbxRM9)`M6>p#)XDzH|3y=LUes9lhNvj44 zaM>#zC#itJ4hpSstd9Q$j^OKd_`x#JI=oE&1tevkus)|`#2bG>0>LLOR&6yg;&@&! z48o>d6fU)7buIhw+fp#J-c>yBK@HTMbWaBH24HG)DG%&Jm+U{k=f*y0X?yy02P<7E zLG%U`10G0YV+!Z*+Da|&7Jy$C%+4hUwR6y68;4P<`DWH(mj$aUwP&G)J?vZ7bbrIY z$buI!ENeD2*k1!RrMk9N5n;F3eXaIyaZfBBr3BG|kr#N#=v9V)9x_)k5FpkfCTpv1 z3c*Ohmr$x|aPa%kHeFhUGd%Y%K4sK-cRj9pWoG$iFTH1F8Aa^*_aQW^{-HL$MWj)l zbLFF@t`9$tiQ9(tor^&(>@IbC{hV9GmO}B8VBV|MVQ@U}@)soMr)mFlvcsD8!GtU( zTyx~RVb_bJqT+(k-jheX=4NPW;co*{zTLVNIK{wGuWeAaY#%RkJht=qebHpQOzm8*oGXmoZyLb9JSU zqOE1ELuB1cuC+6mwJMdLyC}zvq&OzG`6aDj8ueVme4%QzvUnm*t@NFCz9u9zw3Xt# z%kFNoo(c7mx zA5K9XtsFFGoOzk-&%L-S9*KYhx7FfR`44WOgS+m z2>%7SS6!JlX!beYk^1S=N3mKw{?+3`JOs?ECKZ4m!a4p6Q3hN`4m*Hew;#0qt?<~P zm2bXF^|YJYlic?&X`eozA%`#|8$A0Ny$2^a4?@vH@DBLlI-DeL6|sf+3&PWM2EfQ< z5q)Qm{DM^V8(<5K;$2OgRb%_|v1uu>%nu`?y6-=nj5u1TDQ<7J#T*HUfm6c{UN;lw z8Q}X{cBxjng*HeoaF$%$I?y{RvHY55DYqzD9ABK~)yoZQ^w$q`M1iJBa@0ch$=P4% z)s-ArS%dhAF5#NZxT|!#ThZG8q3ylnss8`};Unc3W$#^%$WCNCMpiP*9!IuhXT~`q zWQU?86e2Twgp*BVhwPO-&M^;;^S+)wpU?NVuJ7%-{<*F{+}tXionA4UNP%E;%x;@Uxkd>^QSA4!_rh*8=1Lj(RWHcZINu()YGP=x zD3DLPn{=TC2-ZlpU;;1!-X#J&SrbC2wRL{tVK^WPasZ|S3TzqY3Q)Me&l4kkXXM%^ z88XX1PnYy(`F&VS3K)>)PH3lA=6H1jEn2j@`^N{!rbeQ+SAlHmmuxEI|7^mEI|-c}bMvw;Hprjfe&LZ+L?d5tXS5y{vtn=DNpz zzQZQ(yI6;sJSJ!Z%-!JFfM#3maf;O}&f;EO{rlJ+p;UuPIZiED zf=+;gF*Nfmj_6LW1)8oxT%#{NC4s^Ic=dV>}I-_BPKNhmVDdM@!P9pJwx5eF$_3t zpTQ}~B46Z8xO{#Z7&bg(JD2c&k|Zd7{4>yi>B<5XLonB(#YfrzNnw?hslOk$hRD@V zWK_(?h2)kJyfVT_9HY*3aeNPdmpsj{`CYaAv6Zc;V2MT}Ga{y*+lg3uNk*D|{3nV9 zzFZv8Mrf^tpWZG?jNL683x%r@ZUyTPDzZ&lXqSA5E)~(*D{*@y5eGG14uzbKgH{II zRQUgTQx{QgyC=*S0c~9gV%ZfT4wMVVUp8vw1$4EgPBzDjPn%ltVJVf@y4%Xx!vwhj z1>iaWb*O?=@`D*5L+b|q2of|)KZ1uritAKtyh5J@5rtRC!TC*Uhd>`xj$|j{yAsA2 z;?2I1XcCt~@P!Fw^CI3GZQ!4FdecnL~|q3n8>89ug?Z z^}A9}$fHZPX{8{CEX0!i!H;nD3v-bpW>cG&Hdap8gF9-fjGRnQ*rvQ7mz0S!lBqtG z-7AYhyXl<>q$AE>d^103-@{6&epiJd=FoXt3_1|=i(Ii1@o?X_+fA7mdPf!Aut-0vdrr0=*m zrJbq!&haox(^8Y@*+94=ROi`>V&QDgcg%H!=qT2d0y2sqmc-phFd#UY2jsI?9#`v70aY6MwAO=N&T05`5<92gjn`!k|Q4oRxh zHRIB#JO475E(`z6@P!UHUz6eEOHty=MiCR(aL<3FJ4_QeN5@T*56gx3x{Hg8h9HJY zoqA}=znKE(S;3Iz>&;V3&JspWvl1saU%eA2n%b9s>XVe|sbO8MadiEAo$DO^q{QBb zM=@lKH@a+WEg6ZwctRC_?+b7L?Kqzv_pj{r^1YLm>cal7Lt#&^2@(>4$(fG|tMXu; z+^A6$jA~~xRvn0$yv1|EP7_Vp2@BO5frU`6c8R`hrpNk~xfQ#z9t2(4&wsPmbK^=K zMV#lB4CLsj>y)Zz~L0U%&c#}aVn~QL(g@|9k3X|1ERD{){&AC>{UgjN}|5F)) z=zHbWI(ni56Z}mi6G+@Cpw0QQhQ<{R-E6ZOhCNE9n33J#i%!2q4148WOnmD!;TqD? zWc(jUIq(8&X2l9uRcan#EZc44-@R4&IrUBU>bzqpN8lO1bMnjpZ|$qy`4=xL@%JBPHJBN=!nv-E=`6NPz9d{0$V==-?MUQ7cfX zjeN0yj!MBPN-x)s?EEH7?6ga^HH7|ZISJ5*aEK}N?A=1!&s@**i)}ydVX-yRsYuTG z*1zKlhGoP(NNPOw5P|(kEo+9qoEh`VKOAm z3FoyUJ0@0`nfxr#Ki8^~Pp4=^982C=Am%{h8?52}mYgg6mnA0z4jI zP%pQhxiP0ae)`^w<(Nsx|4zz|mSV>EiX#&JF~q{5aIS3YVc;`ScjVx0ciw&71(rtB zpH64rwJd|!^`V1Hok%9WL>dr393Yq&%p2VE>d-HDd%>C&yF+GRz)dfJRghRE)D<~4qQ{wAq>GXl1VzAJCx>lLyXZ@vOft-9n5 z!vP*fyM4~wvHVtx@3HV`0z>L>ppD;e;T*^KpULq8MIsuuWS9L(Kp9f-i+k;sGx2*| z;(U`_S2IKX;Gyk0{Yx$S@~re*Bz|=q*p!w-ylWeLAw{C>l(h_np@HKHPoJa!$4)dE zl3@)nZx6IL*;w+L>d-dsr4DQlV#FC#!b2gB$1;i&STr?ZU>2g|en7>koR=t;bC|S=OawfXK z)Pf=jKy?f~IrWISfxYtgb6O1l-^63#kwDV+EZu;WTBK0$l_BKyfY+{W1y(L;1I*gt zv}BBK*;l0b!XRB70eGb8`Ny3dVc;d@=C+c|CA-x7TD?Q{>%+YaU(VmBOiEmm7Ra7S z0n7GdPTo=82ur4T8#R&oW#W{$1YB8;L@%4$9o_10<*!s-TxpfB+9cD^Uol7-fRI=I zPryh3s=e&b)IDr|jEH1+wf0Bx=UM3sJaL+(05Z?Q!*Wo}!E^Lz(9KuSW%qD%YRYt4 zB9)-hwekYb`=Xoc{wdfE?d!#7QOYYUWl@=xHWkG4oXMbvcFjhGl)tg)E)qd0#xq{v zaEW+O1BRm4zWhPWb;EIgpAe>{{BJ>Dy3qzQ4nOpTB?(fO_Wr#Bdj@ltK9~-1T|R}J z3*wv>g!x*KR29R|nD$FBYH6GcJr6|ZOF)uL}xh;uO6j9H@hLqW}iCPi<4V?FGC0K}wHHKPZbsKqV5Rq@Mlg-{`8Mdb!E-_U8 z2R;ib6ygrhINZeK$4Oe~*qM)~C^g+$t{T>S;bgQXw(NapYOJw;>IjLUn^8eyDuf-U z=L5Bc*s`)36p{IP-iVrOf7{WFiUUH!3XOLSm~({gf7wcTF(tXM(P1jSECz^VzyLHB zhh9jCSoDZ`9iZ|kd9ror;g%j7?budm;kTZXIw7&3u;=rMpo55fFpOGrX$Gn8(S?HJ zq#~8&Kp>{FwN}bUOMY*Cu(F4T(NRtVmZ{?>0c1&da4vaxiNV$K5y#Yx#T7mA#SMe- z>nxLZ!LtqT8~4BP?uo@4;fOxt&f0&07!mUdtGc>CQD6>3*+qb_vxl>)TKrbU3i+sy#V}R2gnhn2Urjyb-zg) zX(Z7-rbHUlzrzA2)=2dTsp%ZE?wSY5@&>^^K;yjZ^pHm&;|Fpk`i8cn{?ar)xz)Ht#I%FV4lXuDCb`9I`ALC&eap46KHYd@B$HMnCSXPZlDZ79lsRh`N@EJLGipJfItHn8r&|Cnzq?#}nW8GqP&2R5SqMP10qsGI!G zM3718ze#u2%GkLDr$~7Z$;hu|t}dUfRnU$WUt#(c`F*YOGQ`}e+zCO4+na~jsO_)* zo^v!+AB;0q-8fT|VQLA#9aHuDph}uXJ-byw`XpWWJG-HvQTSp8ntgC8Hk>B5u`(n^ zKBM4Z(~9e`bK**)tz?wm&{E7zMzP|5tj?nEp+OFhoeR%o%G?X&`&bh59%kF8{lrt`(MLA~fs zN8UyaQ^n^U``S7!QVwyIln$h=wdJosAK_oqWs>dv_Q_hz|EjJJC1;TX$7zgN%<{lM zJU8mschlU7b`KTGN2lSy<~7^=OZgX<8+Qsf?)s@%o}gSl+Rnu}vAyaShs)+u#mdXb z54Dk?a{=;OO2Qvl*4Hvt zV}X`8uR$mPL6_4UH6O&)Qh*=wTCn)ulbg8`A~0s=kQ968yGm|39}w|dX)yn9_Z7I5 zMX4WDoIOicwo7UwJimoRO6}_t7UkaAE=<%6m(Bz0;QNU`;FYV1R4P=C02vZ4oYP$N+<`D^`LOmM2zwbC&@ze7(-fI-;Xk}K zr3icpf)=s!NxYxCE5r+F{TU5SKF$zrTQig*`%%*SW_NDQ+?P7+dSuE8{@*ThdHih7D|?G}DT_r4V8#<3s$gc0 zk9?aN$pm{?1?7vEE>%a#j@k!Up6PUeg*s(32AkUVheKt!-M z*J9*UFg1ztQnFuOaeL+;+b(y}sT*lY8~WQjC5U-tjQfr>UKZH~+QT7Hk7^-dE|nFP?F5a$W0A9hZGm z0sTJ7vS@n5&M(8hv+)Ol;56p%IXf^ZHMDQ?hGHYro;3u)>IbaFOSO`eC@af zACPpJl| zIxz@`W)Cs!IbvH2Q`xR4m!dw}C`#|2#^{XEavp{~FiP-`^rM(!CiorzZFme69E$C` zr;PADBxqg}Oosi?WYtFJA677Ivlb#{LP&Q!9c#~ObYn$`_}v}6*JM*g9XhW0)_06k zo0W(4h+Tqq@1uBI+K|D48ya;9xVa4f4oL75xYKSl7`7%kfiTzxVF2{anlSn8nb6Vd z5JZPqhaGSGjWI!Gf&0tu#v*3S1?)CgSIZt;#S!5)u{`ntpO*-D)eGhIMkElm3f5Xu zFfkpL*YOsOqoyb*Wnthz+T@94Uy>O(nJ50)=y0ZhS4wO@O*_(k%b9Nn`vo&Gm{Fzn zwaaSQA?1;Ve|Z81V4xmG%2s1nVna$@%GZNJ;z#|Hb&th*vxM88Qn00@#5rXzpRp&h z;DfLS1isOPkHs8!hn$?3Y9jgn=-=uxHq=_WS%w$J{ayOn%as9+*^Jg2q0hdQ~e+6;%dsd_r41iNdsk5?gDf+Kf<$`w!x|@ldMbwX%Wlh(sQ3X4O zN?xoXE>>~5D{+SbPF&N$12;*K|aKBp#Zxxpd02)nOS z)6<)YA(!1*w!cJ*RfziXKB74ULlnIW)`At5@cy$U&V{G``-r)btGaJ|u9;u(&pwsE z!Obt@!k))*eFXC_*V-5C zt+En6V^?b3ayPnkDl5OIj!9pSX10|QtV=cY_DY9%gu$KATB`%Q=pF~|H#pKUuF!Nss!fS%l{Ni~8 z1F&|JVUQge4nLhqP6f>J3#XXZkx)Xr%T}utZJ_5f6Tni#=fC}LgIN{%?PzM;pTj1( z_S#EF+LBsJ^#Tl-_V`=~i8OGCAX5vzwn#Neyh!q?#A?RORVUv{283*fDTrIl^oYD=r#82y|V}-!WS!G2|=6bq>RkIyY)BR9yxJb@ys2pO_VtzHI z(QHZ6clWA=MnQ*ggR+p~lY2w-12TS)@XYBmlVoLZX6PiF{%({_Dx$uch;obI%%`P$ zu|-y9eEx+Xj4Nv;NJXh)X4XbZ7HYLJAE`OL&64N%5lSN1^%>%90A*{&ns{)oO_5^gnUDE}m8|*9mdq zX!{&8Xy&a{8Wu_}kXj$~Y=%j{`4Qz<&7LlG1_=P<9F&DiawTTblwU)UInQJ zEl<^PTfFNcG)kfdGuffA^YIS)k5wzn+t%>vcM+HEq+#|>8JLSM3@WJCQ@~H) zAcceR7gYqBygGz#-m!MU$~-{$(8^hlhyP@3qK8(U;gz)QUHhd3&pUBh?!}^p(<_mTYSos1{H~@wjJqgh z&_ULul+ZVKkuOnfy`KUSj?eA`9OHXf5(DRr%nwT4GK5D&zk;oC*Op@cfxMcW&Chlj z+DJ#{cpgU!Say)9Y-z>6on}wGNWBH@vk_-9!S|ZRgxwB>1%$=>H7;4yi!3DBTIX(S z2z7mf`0~K6Ce>pTm;B?m6|cLRBV!9WL^@knv^5tZCETW%yUlfyFgBZfq;sW{dHLMr)7W>T8s9P|6-;Q$0T~ATZ*>yIekRa5BdTe*eG_U|%IJ z^z>&t$IpHhY=SaRN$?ripr2#q2oYKM0`rTt-toX>*`moSd4}`Y{D3F(W?BJd5@`aT zGOlpDLBhm*1{n6|@S=!lt^5h zD^e`z3~1DXr_Ka}4Y8tezD)>qtim9g#^StGFjYK z-mstCS!k1kD9@_P#!=QU^&wgCzSt?;u3GJHU&qnn($bvj)LGLcb&l4ZE05UYmQAK5 zh%i(amCW2JBYH1r>Y|dFL{l_M!ovw%Yw6dfu>)QXBt5V=>bzJ+Zl`a;amSsWvL+9~ zV6JE6SES~C{7)0b_w=Xs!-4ue*I7vO8o<=3m9=YY9m_cKsJ^y*m}gYj&b!=7!~e3Y zz>!<&Leya5K#4Ei zDBs7&rBpJaF~MBU`Egva_8(;j*tw1$*JY$+z=#1ZG1tX>=j(aCUBP|fbeSBrKUqtL zg)+?9l;KBlxI2q;^tkgw5EJHVUoB=;Vqq%I3jr+jB9uo&FFKVu=U=oa+PR&{;~%6h z@-Jgfl@eOa%|y3DVnv3TBZ>Nf}c6cm(9Q)63$2NDKY% z3d8wW+0IQ}T*M;3huh!UH*@_9FiYCv<*%pI$_0vCrZZ0zr3Op3+?MMyX75e$SY|*k zXJr#hp6dALy40qycu1Vd{N-W$8(~&{c;ka>&U%dxS&77^^i6hX%oSsd8lnDc_%insy35y+^rDS*lTm4J-r*H#*$*1+2z}*Hi}=GS3W2Y zt-k&H;r__-+x`{*8Wq#iWDQA$g!hm0)9x}Fk9AU#W0QP+DzYcBST~HR)z10Pv@7`5jQfcPuZ~F7w^Vnj@xSXHd#?ex#Po6Ys zq_`ON^(jNZK0u3y_$wy|nC9PF7&yIoguu}NrZ7u3#T-<^P3s?sLn4bsNqkT0`Hn+% zoVx#m<_Gs#G`!wE#<8B5g$pv+s1k=!86ZA$lHfiqY@-%3W(8{#v^n5<@nbtvy|5|y zCoD;xVr^G6!3Fr#&B@_)rXAp$pVEu-cNN%?_RkbEb^6L_u!@x+`bp^WlQP5`q#nry znvYSi7;A-nEa)?VWIYT=xW`}-FaCirS8=g#9Gk0^hNUy$^kk)T1lDB}8~p`JNxSDy zv1V{^n7M=qwl)hl4@sPphnQ@M*ksO62SOg?MyVOTe4T6O&mz;QP6j!>6mwG9HdpQV z7FA+yzA?*QH&XoNH7rfbclE?e-qb*~KeCF#qU5Vj?F$BfiMoFC1!=`R4-amcCF z*!vM5RPIf>*?BuF4-(`Sn7_?u=Ti_&hm~xKqqWqlV7cs7x-yy4x#hK0sJCFP2QICv z8z5E%&*5%!0XNIfbBj??t2k4FoSkQbidy~B?G)~i7Wu|!e1R018LQGOp>UaQ?$_$1 zWx2pvz?Cg!@govw(WGqIb}Knf?d^)+qR6tm^aRLVWko>|kz=ZaS)L0+rA_oDg5Vm0 z+C;y6M8fOyoH^m7M zr8A%hs4)*&>tNHCtUqAJ4Jt=0bUr*B%}*Q%FxI(j#;3Zu$TQ{vJeGD9+rKLsj2XB+mL$xF4~CKA1r7g=amPs= zeXIChgZouzIc?F)(KM~E&=^-|#$c<)s~0SGqj|5ecGCClN!*;-l;ug_pVfP1wLq0y zdJQfB!PmxfnlVcM&eXVW^vE8_fC(P7Ep+4_aRGH+r$D+kACJ5#H6dNG?oHZALj~o_ z#xH{?6WPBC1sP{1=a86K*k93HQ|>ZfQ4|SQA8BUAc}WVM1&Vyh)DxJTyi?m3Z2$M0 zN~pLGu>WM(1E9Mm^f?*uDwuCaVGECTUuMxf+l^LamHqPZa!sWwUy6_0^=aE+C!%k} zmq=yKuR#1p0TK`yh#VY~GDa{r-gnPVX2@LrMdRCf!2LL?p!8Db@Csp%O`G37bH$O2 z-QcUEp8|y8NTUOtsK|yHkKK_l##99zcD&nL&2%(vgVRo_b+WGV{yXD&kPv0tc#gLX(rrRdNDrjHu9r7-$EmulBoaY*y8Y@g1gd3aOTq_o^^uL#4c{I#9GDmaPQ7h=^Wns!b(h zGpKuOtj7Juh2?EjIe_IIAUj*fTFjo7WY?<82wr_mq?pfeZ8q_XRiESck>b^Nhk-W6 zcDN)ZuJl>xx9_9)$2pVhM|;zj3Zo8o@&2^Lm-LneL1pC@E0~8(jR>+y+tPWwEHtIt z#+}q-c{(2+b2>JqKD_N{R#G*^?|^8){)lN?Qz~MrF;2sh?v`ZN$8ogWjFX0rL(w-w z;X<36*ECdqKRgooR^g_&Y^|;{qWbsaCd>Cg;-qmlQjbq%IiUGsAA+x4`vPvc04m}M zy~?@+ET{)fKaD!NUeREs(LP>sIqcU$4DY+LpZ!2m>U2Kl)Sp1}i-`~l zF5RVkoYAJ7_H+K=4{`x+i~WB0UNFIxmAKL-o8rgb11E?ljC)g7E!VXkzr9}}ra)R+ zJ0B7M3xH3u$b~;PR#ut6q0{6s^iwaTxP1Vj)ImnfIHwy;Z!3(wx0C-BKdj{**F{VW zOS~VFn(}4hzC-!Qws4+cRijuTt%5iduu5W4PvzJOkb)FFZp#+hvC^rx?&C`zas1$V zIvy#IW>ZLJ>j`0jjn8ngD~hjiLdS%kxi&#uYtQ7@d0rR0kk`(fytjZYWdr}JeN(uL z-nEiGp?@HhC7+EvwR49Ut%DM0C^dlY#$Xn7Yexd?CkRYcO%e0;QF;-ui^NPWPFmVp z$Hjs-7{uSkERJ_Fr*Hf`(Kl@^G|2Zi{F$4y=UJ?!AQqHCyoA7vjRDjmpEfY_r2&Rs z1;lH+jHO!NG3x75ZAE1o;v# zfGf|zfnn2RWa;bPe626bbILdE_e`#~ghQk*73vXlhut2`Vgh0sE*GwKE>_I)SFTN~ zt510emFN5fAk)7Ybn7WfoOWlT6Vo%nE`yG>VP@BB_H#cDY0-R*7-dw5)Z~f}IR>`w z9!qIa@)nEbpIR2G20h#dPpD|j82$83ix|{ee!!e~!UvZW50xk`M29TXI(jW)cOyA( zRaZo;)PK-@&y#RF)sf#5a#&hZu-Q=ZB`U{yqv1`YoS?LMV8$(58lZFx)I` zpS4N#dI*~38n*9W;brK(MXqP7B)PYZ%(FI|n(g>&)xywj)}y5_OHlucYL)p6(6G8- z9cl`0YN(<+7=P*NO|%(S#Ol?%)+4rWk;d&hcMQwe$R?F*osyqCcvkkC-bjr+28_2M zo>v@EtqK6#_V1Sf8&yE1fgjx}Z%~vN_vmQMveSWgyVT&r)Kl+*Jbg?0sEiW(_w88L zlERZ&0-4*YMZH#nnt+m!rt~T6xkyE2b@{6<2e)gD4z=`=VxgJuf9(Fi7pyT+VB;W)}W?@mpI-l@%DIaeM+%H|VtXQLR&gMW6|uWsT`1b9(Db3KL{ay>6@~ zgbcmpdQjTziVM8R$HrK<$P5B?>^;|7Y7Ku1p_g*h44S5+^C@xrF{l4Ymyw75X8q0O z-_8Y|Xfb(_U<+I*h5lYD41>5axG5-zYu@n$oxE|^o9q2%Z4%_C2% zuVr~u*3|#`nECAfna?;2yOC+5LXO%EEyk4m+*hiC!WEZD*OWUT`Z!k?g{ZbN3bR=i zjOEn2ak(~KoVaXj_E*mAFu|Pv8h`$u-^)HC1qj16ik#^B;YMCN&acnO_{V)@7}p4) zlnD0lpXaWCEV)n^3>X+fSr0HBi%(aX3p zEo|lslCn>hLSw@T8d&w2-%+;w#6f^>LY0Pc*TZ2jsB_;2f`Lg-^{Eb4bb%0orMssi z)lPH&vLV^^`dXVxm4N;+;DRKke3_FB||ze zZS%Vk&Fr(&vuj2EJ|VIvDCedmt0moy4@OGRIBcfu7(4;(G)EH9*6z>xduX&+WC!Uo z)@r(xps{=1Sp6i?Ta1jQQwv5s50G&9a6*IVep&7X1eoOl*kkSeFF3nu8|dNnUOaPi z9MN}?bW`0LCc9F-HB=>{q0?e#s3a}jPbuatPI=s=M+(e1g4Es+6wp?;Y*Or*dIUPS z2Ojm_y3AR}1D5myg{NV?ga>{=PVYMvunA8jFl7Beo$@oGabn>Y_Dn-}n`tr0!BU(Y z31h$U{9_XW7R&Z$vyV}`PAhX6s~MmVxm&b0Fp);kC_xcgK&|H_kp%;x=#n=YwSLor`o9Ql(# zVq)maMI#2~)oGVjoK{of7}x^vADB}s`fvkvaJNjgxWH}*&$CdSaNFaHbHg)s`|Sk& zQGdRZL%!*8sed38K0mk)?z$k3PebL>1?7~+CD7FCn<1a(rsn=sGSL25NaHCeVg1(* zMsQ_!%7*~>w3MiEFd1B{L}-pPVEx-N8L?d9rPr}l#Fq3*n(vF4 zPv#qvK_St!-W%CdBDvm@t(sJ6Ntg{@ed>rO!+Vu+}+Dj`qHE~bwYa)LbGUZZxv1O5Xht&Qu&UO=p_1)Nd4aE3)A%}!xUhkg5x zDQOGOl>7mpf_`}h9$QQeAS`I(BTRFR$Ui-TWZMu;7!DGtNzT}*``Z-c@J9b29vue( zMd9%SesFgkehgXypI;2|3uW|cNWMsyV1>fTC%0Q$?CGNHbtaxjaz$kw z&JF;5%r9l{?jUlS@vFz0J)jyG%>U{Tdkp zesd=b7a``#E&emCq)xJ_>#$U5?$LTG}9IW5#V;vS;^HcHp89!A1L*82l z63DkT1U`5d*r6T{dz{noBEw7~@w6De))~HbpC_DG$NIk)wlMv4s!w?6PO0ob9vc>9 zznbu_XP_S3Kn7K-v1c>HIw|_V!F}s#4!6WeozdkCqI3w_(xfx%1ePfZB>pS`0g|Wy zz2J_AgU9~6CKxy`HrS_WV&PHme{cJ8u%6NuD^t;ze`|qGv1C*AksqP|&=Is5(C2}A z_z1so^i|~LyyXpobi<4A*S1zF>>2|zEOD6%J(2*xi>Jd(sg8Q=;wg$yZive?jr+PO z^5oA&ep@=#Zz^&A`j<4P11`2*RwHsNb>Di_tyuVd7#W{ucKMdzyZ;#@d>}^jq5f*B z4*nh?E)2gjh6X$Yq7pQxyuT@sN=iZ)PQn+_C#=9=8~*{Efvo3>Wgn0?g74x)MdKvN ze9LHSV!z&>wLdZ*Pf~x?ZvU}_%_g&M02u~qQyeH4c&We@KB5QP&!Jw#7o+{OHqj_9 zs9U3hgI9JyboDRYtGymBKK}Gy>+Gni4ZKYXmQR-w6yt%l2LJ#2S7XXi4pyxzQju=v z{8fYH6Q3^`6`xw-WGiLx1}w;8swamHC*a*dZ<1^Vc83#=qUSUUkYLj;*z)q}kQTtu z+!y>E%!Og=;=DEV`M1#4c8QzgifE1Y-^x;GALR1n}6_Yb~3K>T^>YfhDC2#2+ar#zB5*2LXAtX9)z&OVqppfb=;Nx2=_Vf&$AP z3*SP*Pi1eUOzZxu(QiW(mctc;krL6LGh09sE3s?%%|FldDYWB z5DHW&(;o4Ri9^4ht zw$)LM$vmBvHNI}%{g*%NN@sphS?WVu0Pbh&!7f;z_cy+nP=B`F;nQ;CQA^h%N*3?} zBkeA(*~N*=w-L<-?~*E<}T2-c`cd* z%BvO1C-kpuSXQ#9>~mHZVQMsD7oA>C-gWTUb~ONYJ8zY_lFDqbsI6a%>+;$K+Mg}g z$7O>nCCatrJEhv#=S!(nyvE9cCC@+LAASi~(ip8i`#>u6RD^cxL*RpiZ8j347Gf5j zGB8UDk|pP7C+8<_LqCs(Czi4t!CsWJUKA6W2G8f}nrGQ@?feapr-RDq z`jN(#9#8EH-%%&quLYrndK72Xe4H8MDc~u(2-^{G2>z(NVAl^LDtFsHpraIN>MBQf z?9vHGOWU-?mbp$uD&cc4#zGZTE(+%Uxx|XbEapaInC+B@nYykg5UwO*tMV4?>`TZM zugIX*@O0hKi$he-Ma!!$o$Hxn4u8ju)$XoVOl#UXPhd;DGQ3yPZANzK%VLqVc<3n$CPP;PTHC>BD)X(UDOA z-!>1%#{?Jg)g>Jr2LYZTY655VuAoDo*#FC??-y+%9zzf+2Vw|N-Rnq4Z-gNIip^jke|Ft-ZJZf9o}qS$I)Su z&ssRi$E(iW_jp1|E+vLRi5VQRl+4rYDM8@PM+}3dl`Uv@0@T&+}i&tNA*x8ZtC)CH8uDszSBYy{bZVzB=e*?L;SumX- z31~!B9SrT*#u|+LbxM#)pw*uzx$>PR6eSD6aW?1T_~d(mHvLJe$+fGsVR!2A;afwT z3No;Btx{0}OIp|5A<2I|aD(t`NIJa*FbI_G3O{5!JahbZGjKWD`&(5~;bT-IM0udp znRtPOH~ML^9>8@E{JEsT#muABM^o%HT|Ym6)$z-RBRlM(-i>>w50WCUTTXwQmScBY zY|;k4cGpXBY#sZqvp4Fij1x#AncXsa&?rMKV&<+*B`%GNLUT`Vy9uNjw*Ygo?Qc{4 zQ$C&k{QU~9D?;PbSyhRxIu93xf^>GmeJ;&TM7?T}nJkS{?soT7<`4ws!@@f}YCRV< zZ}zX8xCZ~DVv;uB{LU)YEz{RBW1fui%JVB;a1uO=XlNr*{+fxDpqCd*po1^7{*FJj zf>Qq-t7=fjL@kkVA{bpodG6FVec(;do0B{ZM6d-+9XVs45(Gv|B?RLYH}x_d(`EQR z^rGG^hlvF_(GxAfujM6+zOrU<{s=;=YCF_H8op_a*4_kpIto$g5wj)lk~_C~%*Z&# zXhP=QN!Hk98^_Gg%%Qo7u=zLK`~MJ0|6hDp{{8O=IAJntPg??gcC`agefi3=1y@1( zeMr!91mXAi9s1zj0TI^f2~L-P6{{OhkIhZedQL72R9;1hE^~gaD6(ro?@6tp4$q;4 zICIFkR3t1xLUR!2ELKPYVGst>QnkBUw71`LJPIoBP&?{HoSZ}v&gh907@$+asMQs& zvn_-F_rGQLr~WyS;5>aQNNji^{)n8a8Pr(RyE?6L)kVIhQYcvNtixS6;pH2meU6~~ttLL=T{_6Z|q!B1%6PFTC*@MBV zHXM8bik98c?;%MLX_PpBgNn-3=W=9UEZ+05HJ07!CDq&vdlYxj5B17QLU#EA`;>ju zTqS{?=t#5`5zTp{@jL^sTDHM-%fHC^%k$^Yqem+>Y58X!WZ?Uxk}`2DJw0$SZS0*c z+Mjz|%mFW%Y}-Y0#N$#jAy28Y7n@^natj>(1Qt{Wd5s~N z;s)gz_BIKme1FL#HYb*ym#gYbRE*wq{s@x|?|_8xoblmRvFasDQ}I-0W1^CBEZf$o zgFop`@gvL=eOi#!q~SV`cmNx9g0~M)TTBH+8IF`Z6}?#zHm6sgJbRnnTRcqV+jJL; zkhizj4+v${k?f$aIxfkp@WZWfy_(A8TTTv>g}d=Z1BRYKCRxE=7(!7V;Npv+zh!b& zBL@751e1isH9=DKREA=>!^23kNdgdyT?x+;sZFi(qYs}C2)0+){=(+JCYv|KnVEwzdZ=hpNQ(( zb}HJn%-bVX3QWL9SSa zNVWPhoAb&lod>wx2friU{L0s{P3@75|7mZM6qE5x+9q&w>Qh-3V=MNZz@#N3AB1!Y z1`tz6e2L94T!QGI59(On>OQ~I{-VE=13u(#4%b#lp0%7 z<3O!W0%`(4GxcbPlWz6`jw`Fn2p?f^#9Z#fcTb{#98O0q;!NT&rDdPYC@#K`giS)o z3TW`0@chJE9uuH>oji^K7jK=wJ$iGLNhg4lY=^z)F7Lj-pbYHW6_PvMJuID&Q`Tn3 z^XO+f_uI6v>iN%|Bp9Uz+&HQ#t~I+P{+fF+P8V_9=4mb z=wOf%Dz3SVRpQDrLC^*J`uYm(fq5PIUcDd96WY$ul#~c#azLtqJ~|&=5WWOYTX-OMhQV>+!d+hdy^iekTS$EhJOuS%b>c4bno`u*Zu|v0WX) zPdeMIMn=R!Zfx2O5#w3YsP>dqk1BL!{aBOr z#njT$db%!w|1G6i;`ob0d2-0PT;-czE|-fTPHMK$2TAFv+27O?I%n7)r%D>$$y8!c z{I`6G;JjW8P{P5x_+z2gn!tJ8RM|C9xY+ne%21g#*Zom&VH<1-mZ>)a2{}e&z?q(1 zGttDoU6l4^V_69F_)?cB_!`?7_wz>rDIlfY27WuE7^)5g+d4sP*g^5;?=m~rm(kn1 zrOTOo#x}VOj=U_H!v z^}wrY1(jjKj@+$npkZ4VmzzMueOu1dQ}rt6?QR&?T`V-pH-yTfam3e3c4hKtQYHD` zdIp!iy7FZkpq)vr%sp;`xzl!9nvZpAgWGDHBl za`oJd1q0J7oIgy~P|@3tO_;F~*C!u#_^ML*$GP1dRUkd{-md#n21AE}6q_Hkr`X=P z+a%?rO2vi6QIbj!ZKErl+7lV4gCCAO`>1V+9xaxn#%gn%w++-jbzgX0E~jh4?x`%p zAqH0eh(*0vVS>ODS;e$c#Cke1y}XOwPl}3UZ>XqJ#!!mAMrIx&Cc8>}WHsn3HxS3F zw#E4HgO0qmWM7A8`Pb+nZ$0lm2uIK~R9? z=d!QpCyCcKuVE@%3^g7LMdx;wBbYG+Rpupw48Q4Id#?>PmL%&a#2Ha`Vn#W$o*@-NrXouSAz~O&$QrU_ z86-(!B3oI;K9(%m#=gtWjIF`U^n5zc^LPE0>w1pkevadQp5yuB?vKy4T`*h+Upe zKVEc*ds*FvvO)uh9`5~m$==RT{lt^MZpCQiPIAHX~>K~D3(#evScR22i(@y-Ad0ER z6tCZXg0%5;0zOb?{9HsRqS3hW)l6;Z%pa!@>BLJbaQi|SXzHmunWGmj4hRwibc&J` z5NW5=lY*!D9_|&0e=wPS;a(TDv%p_p2aN%9-BvJ?WzRkl7AE5#2JcH zmb2w29pVoY9tE8$?Q{0lv*nNGW}~(V2GqF&_DSk~H(=@N41A7e?{(WoKBWuxl{q_< z)rOAky=aSW`#c;ZR$MRSEeum+82{@~frKP}O`j~-{0PqD@J%t}!NDJWt0$vcl6b1w zz~aW&)i=k1CN{)fULSGUnXE!EBc3;gN1?U4dBU!q&X$SLQ{y-LW$>auaN4v&FJV(y zq`75Ol0u~5;wk06_v<&6b>26MWivg5X@vAdkkee{>B}=xXFn9(!M}eA|mJ0LzfSUf-=PrfCE7=C=tG20rFJH-|_~eZ{At0o#jCYsk zCWe=i2Db>6%5B^dX8cSc!7$O8FGv2{{@;H~#do%u(STIzKBi3u0GC!3_~_x-AcS;b zEGV>EC$j-@%Qk%fyt7ETh9F>S3Je3}G(DKbX0SHe4(1IALD~oOjraFaJUi@1tX}aM z_8k|G7`J$8?>Nx$aXrw*AJFG=0h7r?0;SKtG+)ka?n7MvF1q;?qfHNV@ew}^s<6H5 zr`h5g<}Kijd*(~JjcYamn7R=b+@oXGD2ql9(W`a#TE@%HTe5gPIy)F~NP?+ja3Ox% z5^#*5#OPzlO+iu~%^y7QY>5Dm|9l04CfMX3$YI*NNZqvRpO7WRvU%_Sxi|PYncx;- zNFrP+!3x65m-#m3tyvxgZb|&4tRSka)C~v-A^ZA&llbF`rl8 z50{7Z3AQfurJFD0258^=2|2Cl4_N=g=*hW|Ysj*g3IJ;y%=ihx%;BsltmHKM@p~<~ zcLLrP#^g8#dvB(cx*fl?qY=tP*Qw>emu_yI7Kwd()_-Hb0&8V?J{ttnnP zG~#?h;=FjWynNMcu$~Zyr#V6-u+K-}N0j2Ym{m{VS|RSOm0i-Hj{QxcXV#%>2_ZYU zBn@p~U@UpSz~sUB%)TrTxZ^f`QYMgYL~cLqfes->5Gwda*SiG0zS7nW+}o$t?GAU} zh3ns76iP@`nlCuG>6{k2!R}P$*>xH{Q;c-k*pohv*<}Sbt7B+$6!d;4exp?6IZhrf zM7y~ys!4CzX>8-FdQAGFWHiTOdt22V)5qPdb+K-Nng+EuXrajOU{+fJr&Z$K zzx$Kx2&X|AC!dxJ#qvd(9Pn3He@Sku2v0w~YL}$x5o@rmzK7oh*tIy7M*I$aa$lwg z3i|ErwnbnJSn(D!xgPkm47cYoj_q=|q9c<=1KB+auD3_Rqt;Qrquv@`+TWJjO| z)eIFXV3Xcsadb`B#D_Ddd}!d>gUu#Ssx1U#KnA;{pm=;%qXNb;`j+?#xk(bL_XlXO znz#2e?6XhDMkWbVGzj8^#%^{Vdt}hz>5?MZY0dWAwnTD5pVIW&fyGK%%q06#u4jwA zUs_$}rac`rzW)~NGJqUpPRX@=rz94jmegZXlGLAarlovkvs8}ay_UXtSO$r_KZ_1H zFdBEnp6xF!;nxaz`A_QcCxR~6c~~P?tw8mD%@ho_H1PZCM>fG?#-<7N zDriv{;%EZfU?uJ;^1flGc2m-EzlkTV?9olHCF2Km94-+cLB3+*=KVAHa~g8MW=|*l zgT4-Ly!fxYR*Vg`YSc#w(+lBK-xO=kNRQ&Bu3Q*XyKXY+C996*aPFJ|PY0)@G0E=K zq7hwhkDjg7TzUCRn5!6|`~DQ>`b`V|PK0l;kg7X9e&J?`xs~(cao@(Mb8HGBmflTN zZZHbEJPkWwsR#6n5o>>0+TXb;)M;N>?H& z17HyTr@SUxCV-;=2of-vve1wBFp`ayn8eD}Nri+cawgh52Yr;X3RFs}brE6joz7c~ z@98*-jCY7JAJl*9**uTpB9;ZVVm2+-fWBQjW=@K+zxH%sZ-s~e`EAbY#V-;y{mR^e zkMDhsOH92Xj)2gGNQhS-T%o&#Rmm2mu~sBnsikM@ai8+IsEE8TA@%k3&AtvT#(UBt zAdCH2)=iI6YSpC|BS5g(rEHPW=<{Jl!9*jzk?8e>8*vdW@7^%Rt975M_M{6Ctbr?Z zxB|-74b#9e_@)S{g63}wUMoa@m15i$X`K0<=0SRZf_FH$my_O#mzk>Y`-Ja?EZz$0 z2C|MvfnWbP*oB!%>kFe*#OL*4=32{8`xF#SHs%+PX;t7Mv?FNIB>TTxsu*!B>37XJ zxU~0&rAp*UW62kFANNDntf5i(`xj~ynkshHWuLfeCYHXHNcyg0{xFYe%AMt6k?rnj z@Wdhu!SD`fcY1kGPk`1=Tn^|2;)e0t%2}{HG&3+;XPdKqUNvr)Dk{oqUb!}v_bSL= zc=6udFAD3t%;xZJ-~77|&_w7e|k_q{(M7)fFLUx?@Bs{!<+p5B(iJyLYgi zmt%P2vNq<=!QottLVc>IaRL~=N_dcWEU_hJ#itE3K ze1Us4`QMy1xa0Xxl76p+*^*CYT{(I0R?IF^)Mh;JZHI{DB)6cLdcIx1lDgJO%(#`xPMTeCZl#xyn)=FCs@PUO~}2jJpehJ#s-MD|jxKtGzx zHP7kzt~r9gvYe~=*lk5)F8BgY?Fa5F)qDa|5{qOi4aCd?#}R!W;z7$m2!gX|N}bI# z90oy*P*6(Z3qq?+8Cr#FDC9+6b+6BTX zgup0a4Pv_Sy@Lx{l}}7nUlWq&XXmBN^X?x0Q#9O*;tIUGb1anuWHb?l_~9+OUiUL7 zY8(7)*=2sb5KsDHl6}Zjv_$C!yG3s9r_XWpyX&j|U|zf^@5|fM`wQR$dhQmra)69t zmR;bAbd$2akNuo-uihEmOCQvrV`c-#80L_qu1t3PB0G3kPx=6#-J%UYsrLD_F^pYI zw z#eq|}(AKBBYokjQQC>4Ouqiv#VdBkx!RZfi`tJ{$cW$A7LKJlqerQZ_C`7FR(M-*P zX?4)$Ge?rDysW)9i_U}6&n5_gCx7p7P2YnHy@W6@u<0i9KQk%-CCFu2*Zu)Xk`$LH3@VmaIMYe1y1-)4v;7R!c7@# zofvTd%cydVR~L_4qFLXF+vt}|C*Bjw7+yAh_!y_*Hat3AWtVXrow*;7V!L^Nr`-}V zOJyW2G$I)Jkfdm*4woOQTXqgk!a<8F=fz?7H)A7`@Q0FHe8O5(SUD;$r9Ca@6L%1L zu=%oR)w^3$0i{j+9@c~nJmvc)A(N6i?zwU>2%-#K$coAcxLXKD;J3Te_Xa`k|2mgA z5}Zspv2|T`!)R}*GvFjD&8{5}+=p(VTxms+|id8hRAm6cy8B*^imKkp-)+_91mQVAZh`7;dNFnHk6pi5Q*d zB!BKV+?vr3ivz)pV4q187f@Wi5poA7Av)WCe9pY+sutJiMc%S%cdOTgWA}M)=s+0D z^e;nmAxuH?M$C_+ffL9K9pvj9vGz^WShQ}T-+Hqtd$GRp3};|g+$~%bc;H4v%t`}> zc6?wjN)s^G(L_k*%`m7h(Ikn`XxZ=M6PK5A$I1ALaa?=`ZOb4|!;Ia;I zuqCfd*Q~87m(#^#J6|`KU1p+$|rHM4=qFzJUVuW@7ei^)#2UG zKgJCNA9uf$GjKpQmfDJ>%=XsEO?knO3~rgag+A0hyLfqv+I^vu6|20LCH#2>=)skt z;WncOqUmVcr+!1-RG=h?Y<`D03_tM|?XBqHyvEHxIA+Z=$xMBrh4!NkjreihL>w-s zVJDkYEqWwlu2_6Uv?pI$)#>#t4cvC=@^h#?-zB|(jLO_J(?EfA+bWZGVt^1^jQ}pq zd^YwF#h;+eIpUjXY*jhqO8)prS9`VnBIBr16Xx%`b@m^33nyRzS(yK(ap5&Z*;RIC zuzj>KY4>Zw<#UI)IN|4A=ys`lpIT|D$c68-n|Ls7btVGMPY@0oBFV}es^#5ure+jC zi!H+H-q8heW8MUw&@g&?gYS(1H^3IRff?w8?Lvt03QTZMP$I z{I}5Q88f$A@3q=CP{_b03X;nkMAV*NN!^WISIA?+Q~D8KzJ`%8_40(AxBw1xUaHT_ zlbUsRbplqtErKQ2E;7gSJl{y&&%HZo7VE1I`0V0TX}a0M;d5ru|8%EEk#v^MUtYv# z3d8J(INUqY*l5P;#P~U`Yi50I`RhaDid-+R|GJd_Z5I2>bN?^SV_kl!OVIwIE@81# zJTg~s$?AN&s*z9k4jBEcJ^Q5@7$c+#zT z>L=N?8P9VRJLEaUOf)GwV+wcCZGp{68ALvTu^t^gN?6{{pFWtbf;z>C*2 znLR0oO!^kzX4PP0uqHNj59gSWVbNt*94-#7ohJry;*nWoVlbJUUh*V zX>7ib*)qxN{3W&-#dEaKbhR*-y6b*Q@2YKt8vhNg6T(P&eT`JejvI{L?)SlOAU$;g zf7fg7g(8p`?7Sy5?IX(|$td~OGo-i1cosBZhXYSz?8lczq8?dB$Cqjuu|A)b)C|4Y ziM}@-%eyk75-y{}e!;9dVwp8DE-E)Hn@I#6?M8yQ0?>#1 zKGR_#*(7+d*9L5KOtZC|ty>}L!6QK6wkmG~&Go+ir)fjm=rXh9$sBuTXwRj8T z-;$K7%(wV_#Dxn&+B60x!!kde&dH{97!H(mKJ2nGWQ||4{bV+=DaK-A$j02dfPb(D zzxqu}1dQjV0tih34?1r4dh5B$y?$zCDBOBlIqLYmGuOE}EEY69M}_t*>Fpw?$5fp6 z&slG30aVy(ncQ03jOJaK@X{B}+H4Pw4B=qw@VgI#TDwmri9hr;QCsRRMAK6z-be9! zU+AfuHOW!llww{psTH-z5?R)92!i)E^r2eg-Vn+T^@&*lK?E zGMLxM7v<)kSSxzKb7X^mi-BL5^{~9IWTxew?t$A4aAJuYG|-VEXTkanq`%7zfs}T? z`A4XRtwTV-*{=fkn>F9PTt8nLe0NAPojby#50{U7<&EKW_AAfze0N`L4INq+&~! z5yN~lSqqLJ64a9z1? z)vM6kESknj^-c(BqF08&2))s)CSrNKDGs0B8LS!0^s8zA;rxelZ=5pTW$OemBWi7F zB^vq#7LI}3#UE=@M@CI}t~~r$-Y2GWYBXN0Zh`t_59@VgLDM zbl;Smfb;C`^}({twcpaN^+hEO{(yD7X3=8te<2iPtf?tIuQIq}cGoBm?MYOqp!7hO zdC{N9&|Ta9olL%Ezqii{hjTMlJ0F^okSQ^9pJ0jZI>y3)YQBXQjW+x+Mwz|wZQYx7 zp$}WX{#Cx%9Z*b~)M0fw{Jdv-Yw=mImm2Vcv&%^c@sCwRrH<1dD-!L!OWD7lzxIM< zzHW}gd3pbzU}jj{&QHj&;f(?FIeYqmCT1fbeNxskXJcef{9FqfrMozL_`S++w|V`yZU@ zD=ZXCQUy0nVZlRnrNx;HJTtk< ze=B%pp?xb!>%g;RRaRjehkFO6i3qm9Zu8nB3)j%9Mym+X?I&!- zlnSf~&Fn@Smnq&enubQnZPjJPNfvf>7c73O{~m<6Q%QS?);a9~3s(%`FdFoZ(-$sL zPKP%U-=JA3Jl;SYddvAsOQD;OSMdqgnNs#rmV@R|IPw$! zXuKb`Yz zaK+6#zY?o98;dWN2m--PBZfMgZa3qIDZ#!~^4B3M9q_yjWASjreJ_dW`0UQQl+afM zGuVQ~V9yNhz$575R?$W)9i`PtT}=8%WIS}{&IGCM?8jeu=*Ji7O@mpK_Iq_&Zc11A zObYTFvTfHxgAa>y`UL}FrLKrQ|J`mfOTisL9a1GdT)3JSb8}uV&B@0U(k3*EP2Akr z9RyD9qV(heeR3A)cp$knesi0}FfU_;tFczTZdY|PPB7(2L$8JOKe!q5!6MB;Y^{=AC&0nA0-2_J>CB7)lc5Y;sh(rOgS!20hXv??CHU? zRHrwNgT$@+JXvgV;2(cnlP`_WzpE=buLVM6eijV(4%dNSD>VNrkpifq+9EPOTIttD z;8{Kk^{AZd_+e*{hv$TQKIJd;KZ9WNGA$wdmJ!q{`tSg9Q3Cj8%qZ1)d2;AelX_8p z9wqHczuUdfr6tafT?fs^Y~>$Uk&IO@Fc64?L5Sr4sWJJF*Zwy*C@CwunUtF|RWi?8vQmLpQ6)as9LXld@7gS4js4?1W^0s&~I-er<{B{1n5;4?sx zaMhr-b15?%NxG%-=Yoaw20x}JZw5x?0~vP{amf{A-??Bv`Je^0{oAijA2VpPC}J9F z$;p4eo!39!E^S|>VN!5;umdIIbK`9Pd zUUvO%D_AbYPm{KffJo0_M#JdZfyY4K3LN#eIIifwy_5w@Ss6gy+t`!E@`IqBNjsfX zT~u>z*9PSEOVgPL;QRhdkPMcIbb&)xuo5KtGykliD}jnj_jno~JezMyJ5-W2A0*wS zZYT0eY#v21rz?PdlwlK+au)Qdp5uB)q!@t)CFaO7{TRiLw6YuR;OUS4QCvq=*ZI~0 zq<)X@u@X!UcX4R#o&mwAmX*>?>TWrQHX`#jAw>Fdg>E?enqYrvAFOhWK7Xh|NGN|j zAOP?P4&22h(4_)<5vRUscocw~zi+$$Q{RHWj&tRRZoJ-5flB)nL~qKo?nfPj?8c5h zM=r|84c*k8&afQMP1McsaS}hT1C@9+><)n(V1sy`pmLPRxG?;E<6{{QzB>CoNJOOf zGQT=|*LQxzw>{~hyaHBNl_2cA`u^W$@p~$}Ur2J#l`sNMUnXMTj4vynwJXnC^-%1J zDO%e)6RdTz1!e>*M++n1l(r!$5*FW9Cux)d`&t-ae(*chWgt+55=^*E0vtri@>Y@g zZ>vmK%3i%Z6QrwVs*f0$b2>2fUF1sBJdDF^J28L;{7Z&q1cKr-qlULX%hy=Gdnv%- zkMm>3t~?L!@_m&lh^=n)o$>~ZF5MlIBpliS*kWurhGg~27EunqRWnPybk-r?!A7Ib zuGieEqKPHN2KArsp1Y~j>$^fQngGotcKuo0-c}EWBw$Qe(-KKMkn|O65`cwLZ+a9i z^*MnQx8RuaH|zWu?fK{0$1lm}$g?>V!CV}^)2yb$qPNE{`KzJN7XCp}i)wSbCsSMw zyWwqm7*_;Zd)<-d7%oM?`&J+&itjbhz_}bDoKbyP$rOKhvsGgserM(je(lQ4u#$9W zuNUo{xp1PIA^=ynq zUos2#xnLe2f+|JeXWT?Uz9v@u(NV5D;aHMKiK1IS&*vCj7C`HIsQSWgDGB>&CYI3x z5#k`Rn#5=0^|9foo=Lv-`xq;x7^u>^(swwVE;fc%CFRykWI73VW&5kdF(lKh?bwoT zAC`=12tt6H|CI69CHDPwi9cZg3lAtUUQKH{G)yqms_&#HnNCgwyq3DhYm(F`psJ9aD0xMwn~Unvv2_$^W$lF}k zd$4CaDRvnRtg2IFP7kowpr)vfr$8RF8DPO@4^Amtu%VC^euL)?}-I^pPt9trAW z`euFkeyBe5)jab(E%`<%&zM&tl)XqgFW6G1Kx89tlE#}; z!a&n*tkG54^)~Az-cLU)o@$7^;wPx?dDG#(ofe^_Juw2sr0|ITRA>wuXEA&;oV%VXL1}5S)V$@2 z@$_KM+a-18#M0ynPTG6@_5Lq1kMA8h#_4mac0RKTR zhTOj2IE{%kCm@)5(U;0w`X|I5JS|`Q5YsxO*)p%XOE7xLFz_dK3MJqhvip{-y~5|3 zy)s3o+Xn@FvT^^g+w04#z?V!4`7s z>Y<9StB2d>JnFk$IN3HvD(Y+``vXs2{;;I3w4QUv!{tS2$+Ne1c3s+Mf^}_R24u%t zVc$WgctuhY{_~L3cwT#sWPf7aekx5$V~77(Dd3e<__kh+Bpo!47`^6llUaQoY7+h8 zgcOwJi{;?!!ZEiNvbz=;GJsT(j2HY-ktvXV-wZ z&enaxbY?Wp?RI%&L+Xe|7nh|ozpa-3^4E5V$Q*GoGqg~=uUa#AsJpb&+4-34^||BM zA+w{WO5bH2`iy_BetEEF3dXr)>8tJY_@2yps48>B3kJ=CHy>j8R^lkm4f{o9igFr{ zj(59%I|-@0ntlkdl`gp8IjWt6dUYnai5+6R7rd5I?EE<2z~AAVKq}5S6QD37wxV7h z`#8^H)JmU6aLAJ7kw@K)+ef_?%IvxiuK5mq{VvGPDkk%&;G!6(LY?GS{{1QQxAbnn zp1y+-p%l3uZ8B4K5^LzYsPy{f(TWWgHiBB~9QfqtR2Pct%wnLFZ>WC&qiJnxV+Dmt9;3TOyRnOLU8n&9u1>YuFH2-fzx*YQsNJnELq8W8$ND?PIo83MEqz`2c^m zc;>(ufvZ|@MLKcfC>a`#x-_7uDZgM6Jg6p`biv|lN&el#7mji33Oj$#q?k_t;7_(T zAcgol2WXdKT!@yNCWMdfUaux)mtH4=xTJ6B;11(sRc`QOIrHW=8Ow@G!cU`oyTL5{ zsqVjh@4VYEW8DO}mAZw_mJ^BIZ->jWKV7bV{yJ{#3*yEJ#!}{$K1oK6Spq^7El*Ai zi}Ln`hACNrc6> z6W`H^SZ1=wrx}{0^*67y8lBc5iL|4tspk5;bC_4xxfwXl`MLXfqyW%V?}S7@^b2-=1R`Q^XI^sAJP>B;bUTJm-*N3*^7VCh=2f@RBSN>Ep``9P3I-L|Mh z)Lz)Hpo3iKyY!Ek<3{??)HVxeakAWgv<9dL-Od${e*{Y?Klq^{{S136AU_33J@Q{; z8*KIQ)S6FYs!wSg`cd^X+K7g6AbTO2{bxzVx9q>G0P+VkfVc zBVI!GrDxeuyLn`n)QpZLfk$y1SnEBmu}d4GMx!aJOf+V1Bou&&aGbsggw#jRXD+a7 z7M4Lb#r#!Alw-7eGQK|4tjvPtzxz*On7{x2zl?&EW4CU;H=}uGdA|H2^O$MtkmQjf zRv<9Sn+AB>-VWJVEgv_rxoPmp$|zWG?%IoXKLIRW8LdJVYY`FXqc>(#g7Vdn5OH(zVI^bM8>_c*yK$$R5agU_l_Vhkgdp2o%`_ zDaP@4AK*;BpY%Y|oi*rnG*0skUYfD7X_j9X-q0u(>khE|8(`>>hT^-LFq0R>n178{ z{K#ELRY489iR-nTLti0H*3+d1IC5}~$9X{Ano04IGt|4hWx@N7)EyCo@OBY))@u)4zwOR?Z0Z=)uvT{OS z)ChG>8s#nf5|$YDXhd&Mw-EB^rTH^VqX`8@57_CZG8@xnYYbq5zJ6{1K49@Bo$(^) zUgtvsgN{dOOjQNC%5Nd2l;PVq8;}RFUz|GkIK&EEJv)#It<+MS$l8q+ZUo1aMpHqj1{k|{hKmN0pX1BgOxwxtq$7GrrCBn9BJda)6XKK z5JJc`p4y0|vy#oj&zAb@yA{e784X!(O2nuHwpE_ri*>t3FqLAg!?^5-&E!VjUQ@pg zRI|4v0I6lJuT0>yg^%b1@;Uj80qf~^$GZ1~(vQ6P~mxTddM5M6jd z*zbWpqiXBIi#MJInlV2(DH#*;qne(xg-em%fnV+Ypmg?>tL%O+|MpVOrp@>Zm*1jn zg==c00VejD9BGUG?8=2rH_+hVpheEwzCi<$!m`(8CLZ+{9>^oF?h!jykHnwg)MWyK zU;fkMjJl-qDeR}JlWcUX+ zPfl&A<*OXK3$?0U(w`{Nt7b4zA(WE==*S-+fNBcyg=Zm0j7baMqUDFiA!nB=i0{LW+`5 z{{{2J@<$81Ye4p`r<<=$Bo2(+JLZEP&!&g;rT0J=lmem;xbsDz#5N|zcTsTEc;Q0~ zDV^RuxqW&9GX*+Cvt`@R29P>Z)PdP}Oi(Vbeee>z>?W_$gt|8(KVfg&K@d;)^;RC2 zeFKPVYEvg)x-jX8aLqAq01am3hP2=?cDvGb{zluX3(KyqM4t za|%-%4t}U3uWK!|$xsJv!N6bQT9CYRTxo-1VngPKB5no06Hf?&nmTXiGy{rF$(e$k z8Zyf>hk!@#csk?s?&f5Pb75m1Q@i(p+o4j9dEqz(Er zD>&7k3OFXJpwK8`#s% zM^~;RGdx^Lw$roc?MZL3`fA8uZ}J^CRHjeK%~$65_Qrsf0NLvSuoj#I<>d!F7urOK zb8;kEm&ulHSr+)*_gXz2PI{@(JY92U)JT6wvDRR7j*Ts$d-~?YDT?#d2G?@zyOltA zNmcC&=Etw}?15&dPGBj+-sJR+voirFuXA6SQ)Q%)ZB`CkSm2zBI`v1qJ~FIg{3R(7 z`?i-TsHsBe^)S<3^aWej0M^rL*SSrwv*KGDmUL+n_Swv+o-11at<^H;CAyun6jIEV zpXs40AYVZf7on?l&k!uWx8)O6D^24kS4LU=R+QEqts5q)*Dnk2LjcKz%T=3WnEP;ssPF8aq%u(0+k}utXUxM8Cr7Le zr&pe9jCwwIEjlLt4yH50I<^0c?N;`BZqxg^d~?x|e$QJD^-XeQRQ>ezVu7;!I{qnx zH>vw#1jGf(i3#cc(d(qGW#YsG$(qs%_G4WTrh>;2{uQ?PIw5%OS&a$)ww z-J=gLH~E~1F$8q5EqahAFFd0hdS<(EfIc#l!4hTs06$W3l;CwRd@V=}wHE&}@;szrc3;U))krl%CvL9^bO- z+(v22;IVlT9rW6T4L-8Hxg9KjI1AQ3vWpw$(x)`CLSD$^wGoV%1Fj+KabbPk;=R7+ z#`1d2Dbqo1D|M=c8>~QE9?Pu)ZTxb%2-`QMcjsCX;bMZ8FfZ@Ch$>}g3J;|O2MFvt zk5+YfJg;r$ZjHmMU1|>ddf`dB*YaWhedyZBDNS4RvOengrKT~}DC5UR-sl-$`6@KU z^CbPYLpob3WYR83xK2wwU)aYXx<+4=ZO%LyJ`z@EOSf=# zSaQjv-h%iS2Hx>4Ht@}{Fk`l`x$;5>8AV)#HzFf!1SnAzwpv)b&t;-*u3k?hhCIR` zJc(^B7pHbwbkNB&DlrIFxu`NGqSx^O@88{%`OM0vMeNVGkeOj z=@(fEv^-Zp0yf}G#JHZDBz-C^KT~g|7s(W(uy7@AKenp|K8=a4J!&{BErnLg&X6=q zK0i7(KKk?)fuil^a()phvvn(z5X(bV?>*V)LO(eyo2PI*X<|^SDZZsr^z&ZmfVeGA z+-I~7T9>#lc%UvVS?_i<%(QS0`OCr7A`L)>L;)>ehmCz69FlYlP!jGVO~`qnrEZLp z&#%xbdyaGV=(C&`CA5cEqsMIBPSkgub7$zku7W2oAZea%^EM|zGNC;sw>SIiw9jD3 z-iwp$^ph0KjL0jcg|G+0Q83Uye<@z#r}j`tDu`=4!j}g{L2C6|Mn7`)Cx$8E@0**K|9+7+Wyjf0Q0Xe zUH!Lz{r`(MK>Z#_2t4`)PkxC#^EveQkA1Iw*XGm?>8JCm?l2fSj76-Yl@6p9!Lnvr z6XN8TfwRd>R`o7twS~6rC+4C&o|GOCd%nmf*uHjpwErm4Rk$$m3sZzt%b^HJ4FK;z zJMa~J)DS8#n_Ca?FK=vm1HL^63d)v@)Gq(@xjMdKw@X{1tXDFH=aEY2AJKI|7lIE9 z&5BY8RCyJe(d3>!n4tGF4A7f1Hw|`-57;^YR815YScIH54srplSe>&xMyZCO@`90R z&so^|II})lzW`E^Qj1`%OMLejq!jG`@@~S`YQS;@G8Df8(7+tEwYamCVNxJr=IJBu z+MLs`-VM*2PF-G*)_Hv9+QT39P{c3Bp8Z;E2|z-_0obdk@h8Nvw+PcHhUo`&=}R1a z+-AUygwu|kHl(m(=SwRhojdT{{3_2iy8qZ2hkuNO;tDISFpjiz7Uq5B1u_cTX^;2Z6N5c<)4+@&0}? zsN@WaGb}^zesw+gX+;VF5;e=H;TaXouzlt4WjbD^ z*gd8Ru&~ER*TJ7%*}3GKC8_X)Fihf7~%rGec;mph;(gkMEpud1wDaab3m%t^(ZEpN4l1+tOedO zvogj`NR%C@7t9K#Y`Bi`5O{rNVPHw*4p_Eryj3=s+bef3N}>k zJYiO-NN5da#=IdwtfiLaBWa{tHbW%!T{(~Jz0FHfTKxKgE#G2{{6XfSA-yy?yg|Q0 zoC%}(gQ|K-;uhHG(D@0;AN>jW4N3b_lRQmm2!PjPnee2JE3( zHN;Iw=m${k@ioTpw=1|N0dt zOar*Vxrk-V4jCXibC~^kJu#frlXqrutP_jsIr>6;5Z-u&(CGeo3%{vd0Cw2YsZF7U zW9IIVjcIW{+(+@ZXKSnqb=RcKD{OrQC0qI4(UM z8v@{?$AHbT`yh6w7gLSN@Xmx{<_f@8M8n|ZzLN7|A!gU)WA&iyI!Lw+^M(xOXUdWY z-RBvp7pTo+3|;aHQFaM!0~F5#g(2T-&4cRSb7%Oyuil;UJ=@#2I@@J>L_sB`DM;rL zIUy?DjuR$C!{O(OLG-dQ!HC&Nq8?H0DO`8RaO)6@FpFgU{^RG5eZ75f$slK)T19jc z+QJ4%cW2Y$W_eDgwM3Cw1P2sr-o#A7Ri_T7Xi|q_0T-hp;Hmj18Kd;Jam(b9Z4?hx zshs2(?|i|KSKC%TevR>UqM2xB`kfR`W6^X@ciqT8-N4@mq;(7rD(XE+BsYF#>`xv> z1dxsu`k5@-QfAw+czv}+X#6s~!QItQ@L}{`S6P6 z^0;34Mb};L=oDUi+omr8ke-dJV>kLnu=}83CdE#PjY(vJ8(P=9R zhwkTO!z2wuS*o7h82qd76Ang(z9>!|&lx%tw(@AEnhZ2e_SqMAqD+1Mkb{6^!GLyc?)bEM6DY(1Hec2 zgqvjH-JD!X>J{aY@7i+1Z%Ve>&(D9P-<@;N)D?OAi@MPp?D`#Tg#X3dSYT7}YvK8N zA95AOM~}hQ$khQ}Eaup->T7w+`VQj~+!?;?KsoHF$W53D)A8eLq!2#tcExll%wN%|jspc3C9!|72; zIJwm*c@^KGxG{EV_V?qmKXwhpYKm=VV;(6W7UuPbVkui|UHRrt=rU*=xBzy4|M~O3-LU_2CZGlT|FKQLe-3Bb^)N!fFG~gUF_AuVf!lY^BuHdV Gar|FY$(vwqLeIc(gn^Nffr*8Sg@u`!g_nbqjZ1)6P*8xEpI=z)1WZ^& zQk0)xLQX>Rr1YuNr-WegXXRzio{%{u^Lr8qBO@aV6U$K+mZLJl{K7K-j~}!z05=nq z4SEa;IS$ZqL!jIcS{ncZ@AC-c9}nQ44+tF;ykABpW)@cPf@&^+4g!VJ(L;|Mp{ED0 z4guc>=(&&Z96P7WaMbb!<8g0Z#qg9oCP}^WX1*%}cqt|42NBFH`~reP!Y58j%bYr` zta4uUf||PiMFT@4W0Ol(SFc$kY;5gZZo1xbyX}7G;UgbkKmUNh$j4F9F;8OSQlC9f zOV4+BVVu8Q^6zp$uaf5Qv(bGfe8Gp+Kq4NU=l$-v@v2zSOx|WPL zypJAN3}@oiOUWy5W|maCg6DI7Fu=kur95|n@LRNhNcO)bSj7L9Wd9cIKXOd~Y)}Ze zc~EWu4v^l($v!7$%)eHcqXE&bqbd>KR0Hno~#2on!4(831oYV;PIS6N7;x;+PSMB7c zJ=4DaG!ZvS zFNl?|Iz-j1Bj#-HIVU*^*hZhcUwVrBX;`*Bq{15;2av?$(1p|XE9d#KqlzC!GtSBN zfBh^bNcTbOiNr`8AXB8UDZg_l@^W%KmDT?CgnPlmsEZb+=k|c3PEfzYt3+#&%r`NH zqtdaW?=R8ckpl2v_`CWX#BW2F>}0!}_{k6Pi*540m&FICw_0d`TPB;IE-Dx}5s@bS zl;M6*i0tTXAU{q99C_%KJv)ZPfcGnvOmd!*il)bezQC~605ZSH$7`adW$pz#tE~hN zL>!?_0|Fapz+)O9NQM(mZfx}Z5x4T4`a^T3aw@!TewP0Vvmkv~XnvU1nK%NcFm+Iz z58wG@Q`x6tSVc%)v|VnrnT{FKs_;!$iaYWeS>f;xqPacRQ+a$%MPAdPHu0V`lcmkh z_d#=MC?S{A?;p2U9_)(q)MybGYZ$H0{KMC`Ctc!Zx#kgQoitseDpk0l6}AJ6VlH4n z{Uj;IZEu1%$5#c9_1$+fF^#t`xuQAg5vCfYC3yX*k=>3kAd}zNibHoKbCT7EJ$F`# zXVB=M=I3%hvA%e^BWDxdr)j}hIDphJIJEY6EHHScKWb?TI2_!R**zRt6+cZ(1Z|I< z$fLmT=#W{}^VrJcCtG#v*%OvV#Xxo-5F!ID&u@ANnr9illLGkUd{>~J{%kc@*!F|9 zlPcCQQq_?@ez#lTz5^f*>878Th=8bx#|(yP#K|u98toP03rZ<%{3NpyAGUjzZaU#N z0m>|^8{>4nc{EC#tTNxneJ0k^BdBotYeBAnz4tjFHMdxE_Gm87s6x*@x5-C%z=a)d8POu5&E2hN_$n{JlU3yFq|k3p%k$ zj2K_iS1_g<;1V4BOA4Wj3VgHadD?hBse0uRU}?bnpW(!Z!%QGsvN$jS?+jduR$#Xt zikRQyI(B22y-l3Mz+`r2!nCDHiolosbk8pO)&Aw5om$^}U-q#Sx=Y<8=!nl71fbi| z3F?fkwriz67s|X8HZpqoN59-N-qP`v(`l9npNU*{3If*|IhRZVJdm%DYs5gpm6|&>X*Ajf1 zn3qR38Qj>Y3Rh{|OE}VA`UNu2s9~QF!NF;+@8nk+IvmO~v}^j()Kk@}DOy0L=Ik2t zNsTfa(NJE<<_;o(tVPKSJ~t2NnOsVlK)Pd9;d1N`ct0eME54o03G62p5?Wg^oGLG> zUQPz}9ByUbET4#?bX}0GEf-Dcw^#;Ya*ye^jM<+h z-`O=64L0i6JT^Ot;+g)Ao2 zUwZJ@xR5-bOcHJUOI$bw#s$or3vr$rqwt@&@ZY8b!_5EL(>*gJ*37RTr<_?Q+kW_C zwWFsYy)Gg7(bEcQ<_;PiN=J77L`=N3ryghEfs?(teIi=(ahf8-PTw${#;q`|z&M#) zx_^x(HOeO+(_+nsdG2!aYZmS^cP>Jyb|$hVJV6AW`Jo)UL*XV_N;PVs*d4XU5KjY) z`gCCM0fT$~3a!G)37ST3`ODlH^|g0zMp&zC^d^O|0Im5=4HcB<%8?$NlbS|FA%)ks z|FD$M8(X{=&=GP6MZCEVrABL6Q7J}LUI<~GZ%=@%=o?N0^u>#5KesY z+%-Skg%K?nJF5_5NPPDMBZV!nkU~l(%YdHCtRKGQG6LTmM$PHK#0QW!P_5FpVLRtm za`so?_%~D`4INNRlCRPLj3tN>fZ!m55YXt@Ugx6vJKv*bK;Y1l{NL_nx2*b5dhATb z{YPG0KR)FOoGtz4@$*@>Zl{(K5!)j?ofeRPXFu>(A+$Nj<5--{*!2i_;r*aE;0@BA z+DdvoJYf{K(+xYHUTNRQGoR*8AAYby86@oPqEls8a>iUX{jtlgpph*>SL$W*!?c+w zCGdq>kDT4FlQiJsi2%Nk9zyFJBAjemHtC@)N#NBM!@Op@yX9aYSU<)Kjb-|&3r)7n z+Fn<$?aM(cY=woO2z;Hyb%}W$%4V9=H4OYX+1RJ5e1QVS!p1kF6Xr=QO zTKU4IG$x8DT@{z=-!Zu&?_oEjeBkJp@A$)r+jaS=U_u-)lK)Ye1_%@FEgR#8%n4^= z{d`VMik?NbOV;hMh@7lh0??Y2Jk&yhe+l(-3^l=SNBuiNJkbRw#<6ca;=KK#Z2sJ( z%nqaST}Fm5xI$3Ye$H4Y_0s1}N(8tk10S&_3SU4?$+w2#D@%_)+j_a`kbJLtWri+P z*k7YX3AvKZKZ?U{6u{hqsb1JFg7opVUrU^C)5{IC5Eftrb&_*p#V=iK4NPY)bhGH*#`K)&m^M@4~XQ!Id=j_fgQhSlN zmZ+!j)S5@56XzQW?g+3%8^#T8sJ)Pp;6G*}aWg!}+X2>|6A{SS7GgGYrJ=rlXqRcI zC!q9;mkOhf?tId1=r*@x*{u--G0@$(#xI+e9A1kOW)79$R470cmmZGMUbU_9-;vT zmmQ7BiYQQ5mz+mXzuk#bA--qMH)fWT!^z#6!4&i9*xn)mQpaG7*wMP+wxdX32N0)gmW?}7y*RQAaNU=O=Ig5IHw41KpD;fp_!)FzE z-qrE#{(2L{lNQ;>Qo=0}@kPNZ!bp1uvi8XORwYhMK>v1z-N?@Rg|EyBj#Fk-S%}ltbKX6<@SlHPa zC2l(yOJ3~$(!&ERgcCM-&ofwoJ%K8gRWNhXP=K9iWG0 zCqp#avgk**lvV4U@*sKxwKvmM#a{T(i|02veI^Rvn`DaznxyRQ(^4|JgC;!(W+_9@ zUM0QiXXHFQj_STy1!}t*4cHMreBWq4>jt9vBmT&1jU!|gbURLDSeNs)Sla_upSugr ziaPR1)K`e2Rc)DmWz)2p(du3W^V*v{atJ@~wssSVvuwH@x-j^ZG` z>`a`N@YaUI!F~JbPRS;SG>hP~FVgtZ6%Z?fX-L^kvJU>vD;Ph~Kg34{MSMRtP@Sk` zU+wW)z_Mc4bL5++{M%JU27u>nvE&7b`6&y=U&#sZ1!Hz_+yhA5vR< zXGkg@t7v}ew`pN4SwHB#Zt=`AZbe_#zU_V2hXARgebTS2$YO`eM6cFt$;y#;!<*Gd zZ{uucGp||s|Fl_OjQlPG?ZS8>4^9UnDB;eac5Gw&c1fHBzu8|ggfqCWTT~pb)1um3 z%E3O&v^`ai%YU|9;xKC){g-&8BiaYv-nU>C8h|`S1Kt#%Gdb-juo0>-(eDb;H=E7F zE#5f#>_<;$r}IL`wx<@J^*Lm1DG+bc08VhLJKxiQf5f;jwZ5@uo5V?Ktvzb}&Os=~ zAoM8t%s{A!dxr0X!?UR=r?aX8O0!DguS>$eK&7|Q!UXsiO!O%n>(6JM=O>g7GS71c z++!0GnvBs1TTB4=WDZFO#+Hx%rqmX08lb+6@qiu32cjuKU<_L-K;I`C;Df#htH$F- z9(nP$;$&Bq72D|?TBuexA;-8={lBZtpR7m4%|Ml5PXZSLKllEa&O!Zr@-yhZ;2(@E z#`k8u+ks$Cq&c-8w&j__2479gpr&LkuVY*lIE*weYkz)kD*W-xOKv~5Eu~476&O!2 zbS{UB48MHTHbwb|v{@qi=gurHJ9@|WeyW13ouIW2A+J;0;9IAN8B~l5xEnsbC@}Qf zgSM~NLj$0tpt9_PsJB1#NZ-yrxM{21|3fM5;J&)Ij#zI1W?cY8 zrm!RA%RV;`Pu_N^`H>VaS1jT(7SpV)({u|JM-%eq(LsJexfCHWXD!T?J z-rW4Xm+imzvi48B@?T#F)7H#yYW@3%|Ni{{hshc_=Z#uE8_i{x^FF-1*L!BE{8K%X zp*NX*>?|xt8M4SXMdlpds6xkQAIl|jrJy8tygJ4}=A zbLE~cI7zDj#lMdRRA6JVpF+FfTQ)a9$0S4p_UwQ|Fjoh`@2n<*5c8#pZ^ZucX!TP5 zXLAg%rOCMj()o4`^YUNin7O~1S zyrBw<%;iwnz$g$V03+WE({hg6LJ@C!MWc4kVyJ)vick;#pC&7lZAS4nip-m0EAbmO ze7zHi)21Ex2F_~;Ag86l4v^jM*?R(tGJ?a94v-Z7YLW;t8(C#UPDk_60PI)L(yDVg zFQDI}ZX|!6M;2sIvFNmt{|REUQ9j&rdpYk^Dmr8$bzlEpitM@LD(7Um-h7<;c3&ek z2%C%;K~*7};r#wNIJPDUe0!sM;^!S>src{4P7DTDo2S2=m7rtVr)Q%J?wH*3A*QLc zS$5~^(rHAw-)su|a6PBdx9)x??%buu-mxay^eYr3J_MlVb1 zH*={}6Dh2zt@hDD_(x5JH%nB%33%mNUgh?uyW5&o%nkRV%Mmm z#P`1j-6d*cE;Xc{a(zfwUxgHc`z(EkY9VzbFH&+2^}c(UYuus% z3g7e^J}tNjeOHv7n?#ReaaUCp+r?w)32)MD-!z3ug^AvSygJY6q@$f{0mZnoX66yc z-*mDKc8cPsq>4W1q+C4nj;v?~g`VQvagp79a;0lwUQwQ#!V<++qdEIjN z&^IZsLz#({tuU@U0%PoKj9Jac*fwwG)LXUg+|QscT)EIx^N+wnN+zLU7Libmep3Ig za<0e5M?>C#Opcw8%DW3##N5cv?Al$|ft{c1Q>h~KdC8?*7GX5uj;r{nM$g?JIgPQb z`s6kwFEm!EEMp=q^*X;vXb|P5@We0hb@gXzCPR*gWVrlr#{I72PV3bUaiD2D3BO?&M8Vo~Woca#W1;Yrc5sl_W9!)ZFvSOp9 zZhRk8PWVlAwRamzKq>`S9t65P<;>ESb-3(9P;GzU&?{BJe&wgCaeLB8N}8ehbpYBA zvCZbL%^qsE)7+~xpe=`HLED9nvQtXA=lJp%4cv^b@NKr+c{sS zd^e{J*B*Vp&>KASe(JJcvE!6uuhhQMA6q7JA8cR8N~M}hJ^Svlh8Yiaj$E*|^vw$(So^5O~ot`lO- zR^OS{bekpV!@OY(XKqy!E9aQy-=zw8UYS-LD8F*;ElW?zSg}OXYzp!%A0em3JWTt1 zIWZ>xj*~=XjOozM>WfV$>)IrjV~MRyBbKo}+{3ewXWbC<@#p^`*bI0fcx_8)cB&LHIRMAW zHmP4Uv2sd8Jz$`vU(v9*YV$@dS8jxsvBdSGi|O#n!nXMMv17FZv(?c?EZf~dta8`0 zZK58uanu4?z>fLE`@IchR1PP7hE;7{tm_Nq#|LG{5~95}VVQN>g`-XOi=$1m!~=XG zg5U;*NDga1gZF9zxua5R-KVS&;1ATh{F_Q-9t)ON@Ff>3k zm(!BSS2W#vsNlbGv9Z>u%A@&i&oKt>y+?PCQDK=c2vg67v0vL3uv0TEO-EO<1e)2s zTI$<}&y&(WN&AgywpS}&ERT{BRTI&@F#n2eUV704WVgLyEqjljjA8o^Vr(8=cd#`$fDxp2jT?`dPnS}8yDA7w^3Y?L$s)7tCS z3KkV6rko739V&`tgv1;eIwg2=d6W_s83jS-^8q^mFPCrYnWy;vaD8gu{Z2?+qSHgM zN6=a_PeKU@x~KiFc9ZCq6ASKMN&pfjd2xd~KM2^j(!s3CX;QtW0hbk+)u%F(0}<>?!%+c)e+n3lI9%#CMh) zi$L+kQs5yRa?kA(v3btQQ#nOx`t4oIxDBgjtEAkde2jtY{7l3QgzWZ~d~vYai`!2B zoz3!Kd}@dPFL&F(r2IZLL z;?(+r&XfHLcjt_mC~Q-JHoNC(8d+4-30O8CW05|%l*VSpJ6EC9DL7vW1T%Q7nU7`t z3J4a%F-fHxMvDxedvoq|rrCWBH$Gerz7KvBzT{+1<^*Yl`LrGE+7N!#niw>vu|(Dx zI6=+3kXicXI@Q>+itfwoGi(xLaW3a|^eIX)Y9WgFY}Jzn6r4kp{Lf$*ZuyAw0OCHg zyAGp9oTgax!xxSeqOesc5>v?zU}>X$ju09-`$NsE(X?3K7uUzfy0Hso8-&`lq` zJse{D6wvBMTSV9_#-4~8nZGY&Yo;q+1&|IJ_I`lM25-&TrP2VB9ClD;+i$&r1{_Dl zz*i|?3T&IpnJimEybO{;=FKyELKR@zvFuRbgJQ-k6caV5t>=*>Eg9&}yMOVp;9mgS zj{H z(_VV#aL!oj#_%a+Qrs$4tAGaFrvaw{a!q4h*49Q)NwKhf>|1v!-<5A>Ch#kFPG7PE zJpWmh5ks^z6t7WF5bZo{g-^k&)`uSKP7EHoL5wg0G3K>{*Z4{5C50%=nyd$i!P7B+ z+y>Bn3KNo4Q$!q?M*l?Z*kPAELjytwU|V*L2rzS%r2+Y)VVbz6VoJ33eqv}k^3bWt z`meAxm-GG7@BAGc1k6NCr*LOPFt?j@%rE|ZKP=o$Rj|h}&(t-@EdlWriLav`2Q>g& zgqqKpLmGun2>bk83{x(Nw+}cGJ$U%-b^lGn0w(&yn{hxU=(J1GJuV5`aeHzodFX>a@g&22=kwX zs^3fV0K-exe>i6zQ~#^xddlaKS0eK~Y-KpTh|tVlpKC4TM<45lsXYGKedZCS@x}6? z#&3jfwbI?!z_SCvS3t_L>9{Jn7ty+Olq@@maz)08OB0OZeb`ys5bH&)dxi~3wqer= zAL6UHlQ_s!@!hR}BQ_CCk`bJDNSu3ThRPO*vBh^}y^fW~oB4}}U?fuBCPej>@#eII zDN;|9?-0HcK@zHZ$S3pBs)lTx=kbWCGgF2?A3R`|nm<_?e3XFcOsAe665e)rY4OQ@ zP))v!?~;1?L?H06uk-;@pz83mD$XnJIHo3j+C)VpcHpMNy~KrE*5bJl?5_Tj<_#m7 zj&JSUN4FfNJ`cOPBt9@^V}|P-0YYE>wcLjIim4W8H|HgKb@bnM5+6pI)kKd=yFF&# z66j+y^A|+FFTo)vMa&vwaE}pNlb1dr_${TZ(hkHjG{Uss91put+oVFiXktkN=1w(v zz7_kx8@cAcjPwmw$4V0v!XL@sS?4~ZP%4>iU|uhn#YJgDj9da(s@cvgFQF1NWWAWvr?b^Lx)eW(8wNY^+Z8?P5BkHN{It?8D9_PFg6$VA%fxsv;dKP*Ts0cdhOG`AQP?r^7(cy_;TO2g-0TT)P)uEqp23Dz%=;I|CtdY8j^vT-#PFN%3&khz{ zyxuzr*S@fWwnsBk5~~h-U{M(loa}p899OQGLwq_jVQD-Y;^Wq^sGxR$kD%ItN@zO& ziI`Gmv#j1$@0k|WI$^+GD|k#Wu~QXGvs4$-;n4VM0CF2=W(C6Q)Z(OfVCl-LPux-cd-8K*p8M)YsRhi z&5O@W?}rHidbu)vf(LczAqS~A*zS>miLpr=Mr`0 z7d&L%WSu$*J)QLJEnScsLzt!)RlumtoFu%Cm4Q3L4*$1eUSnm1w zBGX=ew|V?HaIP}xaU3zJseC|F>x0Rm@XC4SLWh^j$rCP-H>L@!kDuKa3&lMnS};CI zPK0qYc;iY4G$0ri6@*!PL(piOmL+hM=KH7aRtvtVR!uRN5qIWVic(ZB{*mLXwYqRO zi=?q9J%r{=^yxLWH{Clj)LWP1)%{CI@_OcO%d=zxFXSs(hd47|*XUAG8eOCP+JmpB zRqS)E#rvXmBMUP##XHF891+b&P3i;A!#;JpBBn2k)(dKCfnvn;z-tGMmtCxJfG z8S{z0lNO-o@D7k&OS$h`nabH8X|2ugj?KzxLvrXAPMUh3o@thq^GM~sdF5LO0Dx@q z@@3GeihCUD5iwWrh*ta0<_0nu6@R36%d#B=ZKsRRiy}JV`1;UepsybZIglnIMljXU zIetZCiF$jH0O_-i2AUBn-8NV?mo+6MqXpvxrUKQbh3ZJR+s$~g(KLQC+u6TN)Y*{7 z>+37;@XF-Upxxk_L(b*>`>nW=1J9qPvdNY0Y~N~ah1gz(X%XsB>ZIoubcpVps?PRg z{@X}yss2WIvGz-MSGK!<0Qb^f>+`fl4qc#YszOI$JQCG6CULtz7WTF}J^biTGG8KB zcp6wY?}05Ds$PT!oSXo+DGgvYSa>OTzVqtoF|}sF1;ijqIP~Ody=pU@ZFNE|uI~!g zrAE;>Qrjq#Gt1WvBiN0OJD&I}O)^&4L@eUHOt*6&ppivt*bDEkR;IjaWM{SM`?c(C z7gDXa(EMrA<=#%Kw~qJ;8o*kJQ=qVe=}Rh#pt5VURx|(Eql3zc(LVOV;fIX_ll5$v zuz~B3ZI^^NqFK{7O1$!61CoAKRrUOqftR0eDfHu)r>C!{tSGeuA>2+@hhVtsW+h0E zf?Vz;XSDU-aGeF?7iCKWAT)mownaCDS)G7W=&zCTa#3BZW+2CQ8$>?bh$ScvGpYB> zjrh87QZr$c;9j&W?r-fbdXDUi{)vQR8>!=%Eo*uB8XqS$A-z%f-y|Ok1wwhTvDipF zF;{n{z}?aMcC$;@)ocUH9TtEryG?33q@=?aqKST#)f|dUdYIO&rhAkc)PgyFk%Gcd z<5hN^EF)DzCox;n#D`Q2mFchnU8YZzSaU*Gim?-Er6)op;Up+tG2 zb{=4t6GPpCLC2dKKj(GG0+!L1exiqj|G(n{HBILQIuAKxvQIj)Od4b$nv%}$+twbc z>(|o7adb=>t7!s?YSI$-_b}Cs@(+1__@6gk?+KUeEM|0d*wT9uCM^b$)Xs0L+4}(A z<6IdEE9*t!N?W)JA&C(baM`hQFu>M7v2@0&W`tX~17LHWRgV#ghko=_TX36bgj0p% z%|S@$;|}ueN5J%gcV>c0#x~9 zQjYpSYyd&7hp#vb{L;2#*1&A~xYrZZngoU<@3BPs_wLEcwu4?ET2lj6dZ+aGxa~#P zj2_2tM!Di|pTdPl&>ECH%tAa7P5o>{jjn>sRfG=P-P;?-Zw!~2u_9%pf|5^uHkw6t zc*^3$^Fj*r^HI)sxH|WuqNI4Q?;@ze!eS$XIhEbKEv{+L`A?4}`r?@xQ<|4MLQiX%kA7Czy8|xH%p8iesz7~` z8m65@>?JH3;}LP+!xY;1`1-1S5TdYO6;MUnl)1{=f$pKG%#nOjg|%OEZT0Pw3KfP#|668t`>5o?eN_ z<|KUdF`?bFA9}Kx=_DhQ#^F}BV95%szm0}UqA)=;;B7hLn=)kItr?ws9>%@9T;;wK z@aVOUp=)?B4XBcOuf1pQye^qEMZ%(|Y$hDKq-v`p5YXFh3Fn|gC!0hlDX4`?a7t`J zCp8l3YjMQ*(WJ}?9({K&(a1Ma*SU$f!rBPgC*S3m*vGQ+_@w}m$7I4ng(d| z=j{JPwUo)Q02Y8KpkZ-4mnCOB5Je_}t1|;OM6i^5(Vt(sGA)DCdvJ|crVq_avebbo z`7iFJ4JgkXogN$WH}LT`~8aFIZAHJ$*n6F9()yS8ts;Q3otsHh`N2l%$31cKxgu`h6cEoS?vvj zYE@8x8sk@>z_h?6)PWU-9>wm^00DpJPP-x1&rv(}1Fn}#6P{j+DUwJw(}-)Jq!rSD zzNkXV0TN%vxaS4VaAMAjG$&0>4BUAS+~jJfcfKHL-2dpR1>?f09WbHPf9USzQ?q{7 z;QUO(FNwP$N>-LSyhE4EjjT;&U0v1a^>QN*5R!z*PLfw^AItfI;t8#7GwMbD_%E4V z(np>gGlZx$FZW}8kQ$+ruq}xtC_T51*9?ahcHX8?NGCUV1q~8esUy?TY2Wm zw4lfO`R=NN(zjdpingJa^sYES&utSZVhEas%{Z#;75K#OceETzN<)B+pZN+RlfTnk` z8h;{W|4aS!>KVxUwpT)$2 ze-#tUuKbAs8pZyJ0iHuhR{v;@{@ozWYXNxR(34iwOX;A_UcZgm3Etf+M#o`1%@0o7 zfffAx-xd6R#xTU+s)+>k|AGN@V2`dMkc4%G!>FoE0@@4X&+`eCUgQ$bT#Lq5CJuTa)<7JwW2r7Cb#W-<4h0}Egq%#4Tj-X(Ud zo0;kTVB;i5IaT5%!J;pRGHP8G6I0Xk-XZmP_Agud)Q-ZZkv|>Z8~2$RuiQU}HweP6 zUH_(i8d>DbyR4Y1)ZA8%ip}K=A!MvkW^?%bBNlBMlBnNT7B(~QzOxyaX-GRA>M|o1 zxZA5gmotw2QqNH1QQRq>7bIWlWsE+Fcv9nhRpz_;EWxS$Phxv#DoMOHlNF8^qqrvnmAFK5}oDA;-LMW+My7xyo4nR?}7 z_AK#`>CTY$qEK`A(zbX=APcas@RtQAY~ssmbVsYa_-|wG4s6O-%b)ig5YVr)7gcj%_x5D6IuJs!bKyy{>Q&k@-5csvRNf28eh$3-{-u$O%PGL1y+*L zmnm7cF(QO=N5i!m*uRI&vl7RE(oAsTho53Wo!s#=r6by7{;j(6Mw0M?&o#!z6l zgYibvgU&ryJbUTF(VD&LmbSTXN)E+F&4(Z8ppk0^K~cPrF5)(*=`oF-b4!mr?|fsm zVLHF=^<;WR5O%Bs_|kDMA>>Fv4OtN^_|t&DD0_n?XeH5_+;f|(MBPS>!m)NrY8P(K z&7x!eeyef}?3`re8XyR;9E{R{FEc>;-hHZkUDV?QHt!#u9-NxOIp6kBRMsKLsQ5~3XFbf{tDyC$K`HbhtSMk)fknGXSH<7T)=JIraPR)N5 zD%)(E;ck?i3o$)}J7GV3K6_QYr5pILtj4c_I5|i3CKm@Mxh|p?kK}$LzhiPX=7htTnk^Q0*V9s5a_}fkyel z()}ksf^zbpv^$#VGSUl^y!K{&saSmehQ=R9U%4eWzAl3P zfX_V&)&RS4%UZWk)RPUR+Iy3h6%kt<{u}bQbO=2nuD0fS?4h>UA7ImlOC?!wXzO8o z=9fLl5_J4`lsRa?sfIT(t0pMSIAKLhhJVyd*-d=syQ#HGm-F>nlJ8S1-pnp z&ng^jmZ4^^pa^^$!xjt%jq0kAWaaXno?C+4d}Y-Sb9^JEg5S4(dy&84gB<0HK{rvBXT#xS5xyRq7Ro@t0DJI8oJ* zfQ?xgR5^SoUwQUX3KP%8);E$5BO{FZw~?8@FqcAkmEfRbWD+>>VrlQ@v*Da$GEfjY zWdt+Q0I-RGN)O5=g0>{0W)qKo2^1V8Yuz2tvky+~(l?O}Y`E?=G*N#?2Kdb;z`-g} z#;V7Eszf8B*Wjd0{q{!f-;2g^UuPK z|33>ivd_!VbE@Pt1xD1^tsXFD7XYEp|5`>ACYJmgNDBW3k|FQ^4l#BhvwZ)ha?@z` zul6O*Kiil38Gq2Fgt;CIlfdidgM}S^0%onI6!YZFEqr?3yHA-)Rk3`NjhC6EH7;0B zF}if}MS_Z1Gmkzv>__2Gnp0Jgo*>%hLAA;gSE->pFf#Kq8juJgv%g(5@V|JGJXw}=wG_@M zPFc@6-G!nFtv)STY)LW^Q#MWXm@9mu5^`!lq13ZRZ=)fi{JIjDS^~lEXDVirYM$0) zFkpdUmuhITcn5l*7#9ZfHessw#DY+uV>3#qWq92gle9CbFPSL<5- zvfsrx>KsR8#r3se1gC;864<a9XB<8Ku*m=H9 z?^rRcO36=|2|B}C%y^H{EK_qaA$Swy7NH%e9aVqOrWmY#pHm{)8r((N;qG7Z?Mb!nY*9V zIYad$SFHhnad=lvO&LN`L26R_QBW0>IPo33Un_RhT}khwIs2rd4D0KHOT~RwPd!?j zSjm?N39zHVTJ+Nr!{|~p`bI@I>uN^!`6LrQGw9K1p)Sc+$$v!t2ptvICY*dppc50g zG^+4ul}X;^h5V~tRt@b3w$CF1@2gudh$6vFWGkweUNRAqP-2Z`hSqgv^ajneDLgi6 zgE3U~F}hF0+LuaxO&@iS=69Dg{6kxvPOMIhLwyG>)Jfp%@V=BP9q&WwYK~slvlb1E z7=!C0Ey#An7@P>$jMEyp{(7u>#ODpYfGXFS4hw?u8t9^!PXJv@%pDv|z}5`WwePoH zj)wnBFHzxerj16%S;~V>yQVmM%FVkk>>rBC@QwJULZmrNW_`sfLRG=O{qT7Ol1Q)t zNSmpFM+iyiJsm=H9nOnm^;6a^89Q&5rEuIMM&XRqw6UACoMNNfJMSR?2)I{pC`atj zID9-*%`Wb7c0pfiNL@q0`RXx%sxnFDqU2O>G7&e1kOZ;au%tF6eZH4f(MGPa+x(Vc zQ1P^#pWtyiumn8>=W4v>NpP_me6;?62AF*uN|%b3m5qtiz!}|getS>t9tD_h3U2Ad zAre1qO4P--2hKO1k-C;nvgLaEFxuK#Fo+w>6{Yf*6lD{`;idp2hvap0Ide}0@P6#0PmB@u?M6SOq-i2P_@n2l23N6O z0p2cwd)|H0iki+B#=N|6k^w@iVM=`}YgLf#La>tzuV9Opj(>}7GzhslXvC^dfBx=w z5kLBJSHUl9S{|>7=)()dcqHLh=?7wv#ZY#iMOsG4e`zRe62tcP6I~@BLdKDX-3kzY1+pN!M};VAS`7nbZD8Hs!|3(T}WPOwu$e^?zwI zpi*kRYFWX5i@-jWC>Ja7$V@C&!}#87MTp>{gpiXe&;@{+^Grl$r!UHGIhaH0kjxmxSON(04Na+jlQ=m-X5B&H45r!_h!#sxDnn(y*3a!OobB+lH2O4JeBM|4^+vmCF@bd=U+TG zzXHwcuVj_%^f~ol?aX7#%S!xDOU|q*iQ8u{W&!(lTR9*%YmAHy{e&S|@dZ`~n6HDZ zg_J@Y#vVyI6&I!r0b9kPy8?fsv_{yWR2@4%N{|tF&NU7L*5D3x$^6f7nxk1|iGD}Q z$~^^AFV3!IDm`TuI;|^RzT(k_n#<7i`KuDG*Z&Do`ovKGK`8M57gxQ6kGplGI$o>^ zjE-fmyQ>*1m-JOjZ%v4F6EQjs#kUtm21~wM5)P0*@+!!$2H|XSOZ#UmJn<1PSXlbC zIr)%!5#&5j*h3IbS0atEy)&N{$+1*Xu%fpQQeU^gQwD#>yAS?OvHkD$>x;~CCfp6? zCYS?n7K|&fzrJMj*C+qweOZOyANl9|e?CEGfYj`Y!k-`c-+h0% zqY~E(H5lAbVK=g_h#NgJwZ3?2GYnu%HGzun9?TK++wK!q*n_ejK7iMb8;xLY=%v_^z8VU{I7yy zjVBl1K1p$zZ2ZLWaJvWydPRtUtcKtlog!&#Lm*>j@cA`wegST%s;OCFZv` z!_8Ze)n^KWa2l+R5B17TnSXK{k4BN(e|(XAczw%=7s9#za_)!SW}BTj+Wm$m$-nx1 zsoW*x-IwE^o~nt8Yuyi#UF2MIw4#RBVD{%QEf|aX@Jt@ebtZ3aGm_0`tl}PliuW^^~phthNQ?*zUn=9E#b4bOsYrxT1b}!O!B-wc(34fe}ny5@07= zk;)$G3ul5N`Lb|Y*_%Qmx1#YQPhT*f)4Y3MJyo|04VDwY);V!rtPdKMe6Gm(gFGe*#`abL3zom{H z9|1`mF{=7huqHvQ32bt_@AxC&c)&uvlmz>}-nv>JqgpO!qH}+>dx7!%;AZAj)HN zbU&$?*lefcJp)s*B#3!^$!bg3tz9C4o*>f+Fhx6?^sx0Kf)F7u@}=wC`|Nh zgqOMVkM;%%+V|YH30o<6@_%vno>5J1-`8l6rhrC}-W3p~h%`Z3z(NrLMF9aJC?H^r zbZLPg2uN=NN(%^xN{L9Xp(DLZl`b7gsDXr>cXQ79{omhvKi&J`jyuNvkbwn~hv(UQ zuC?cyYc8j@Ovsip#h+i7Waw#U8a_`~@EL9QxHd84TRT$1-r}*urN}|W^Y8G!(cW8m zQfCk*LtIc>Pr zQYI($$OMM&@6`l5g2*}cz$s$>s@8kA`bs6Cv%XEw%f6eXY-s$+(4phXl5Wi`8nXGa zHj%!&v0G0*M|^`}L2=5v+9R)IrWL%jroX&MYuv$Kt!=wD7~A_`8h|MV#)rCGFp4rE z@ECmh<>lcKXHmbx1uRQ^b#M2%nd@YHiLzz(*2<#uSCRB%aidqSc z?ySxZF*Nrf(8e9t0W7iz)NqF__fv{r2u6Iue1EXA&y`KgzF%q{ZJ>osuY#*)gg9l^%}wVlR~+0GvUNnjY%CQM(9)M z3m2-5cc3_j3MD-wIzFl-q(|pZJTg#A9rqsh>c5P-^Zm|mZQkgF2ClW7hPaIQdLYf;4$Q8(lci-@&os4$TjDSY;mZ!%6JM?@sp{k;k^Gax)ri!h4imK%sV-&j&b0 z@Q}rKlIr~~2DYl59oQu)+63?{*EF!cmw3aHFQc>tRc*_0?4$336=6c;kuW`iW|ptD zd~FUsnCI0yq>PML$~UMYPn#-C>>$Y$|EW4_HA(4JW{%};kC%(r&vi&p@xO7>s=y9< zSiSA?zSNG5Eya$Pn{em!?p+JaO;fynmgf^%4x5L3VK?drcrAGGQqXR7Z@8sE@@bj1 z_U{$uDWQA}63$pdox`0?3xy54*#aAsQOW}grA1xFHP4#RtQ)H>XT{Gbo%Ip8ZyAKo z+KZ_pXhpt|Q}xj-2@v1kdYagua{2s9h4gjHN!M;)39tE5h@aJjC;CxkW!Oo%yp@Xy z0T)j4NHou!N!T@M+dPGEB&iXV8pTgttnPnWN^)~PyC|7RF?S=?D}3_xHW~1LTC+bN z(rgR3F4m#>Vv@!0uEweHYl?bA$s|=L2+*Z(?~{0rw=wdcZCr%5j--CrPiFo)x%$dw zyv5h;yZJ#N?XZc1;wl}Y2R55m1I~Wxd>q>#B@4s)5!nc`6n4AdD^hmf***4L4irv? zNXKbv+)dqCx2GOJbfBLb9Gv$PIRrX^mAgi(!@psrqU0DH%Ms8(eG-EZ=|HXn?E?-@ zXqFn-JoKb#x1*0E!VAgdY+{;>P# z|NbY~fB)tGga1EJ*sB8`IK%*7g{EY={wXz#Y&W3bKio#KZMA*jU08}r*tTVb{3A(A z`X5PJ<(*Z^IndmS_c#iEf!p!E_m61uZ|kIEMY?Cm#%+AhW-EoKbZAF0(ZX7H3ROsL zaO(Fv{`HeI)T>ja8$0*tf4_K3(C+_{AC;Xa&U9Am-tB8REixQm+~6lw$)U9O8ln2* zx~!%d7thgR#23fRiV0u+C1LHj9IdxrO60YOU46Z}Jmklz$B zphA=)gdDxb!hrS4V(#;!dmpkMS$q<^z@VG7OphX&bR1?+7drwCPV#U>I^qwgpTA%} zSFvPuv)u91lyn3TmfDiU(e2KCu(h;aAg-z-V%dm13XK|g;;Q6e{vbO!@1^2;aWRhqmlov(OMz({u1DXu)PVr(r8?jgEB7X(I)GpddUMig-U3(|H8F!4_X4!BSNW-VjU%g^d+3tb!3PT}_h zB%YPvTT}QVWMLmJ@};Gcidou5?ZSS`Pe)BdOY|()^lC>=Rh6Ny7+jpvd3be|S}ex7 zZ)TSU*vFYP!OKA75C=HspM>o_KMZda;6g#?HeRZV-u|gD-I0?OWzR23TxI2CTKU*Ak@RdsFgy`L+eH)>v%{`A^2`K#}pi=Qb@L`jE1icI+>fsE&Y z8MqwqNX?CFuiePx@w+y&q$;g0kkFQay?c}lbZ_NEb4nFB zT_*NX6V+)zh$o`30mYAB&WQs zt5Ng>g~oK{ipp~Q&resbNDj$16G{w9_Nn}ZeZsa4BF>SSL741#GXL2+$&y*{LqT2N znd|jv;n&stHS>-y?$Ni%AAW($68Y!%eBkU3AvazN4c}^7$G&>m_0(Wiv|B_r{ljL2 zYuMpae}=8iaUY7dUFctsU**T;*u0_V!F7i=-XhzG)**)*>?D(Me`znzgL1nFP_mX1 zUzQrprQ9(t0&tWBc%pve%`}o6sBV<@9_^x!kKy0Q3@NjDqpJ@1hVv0#VK0YkdKC#< z@r!XJrTH=skBk1Qcfz*e=TI;40>Qf}kx~zS8cay`j2ZKIi5RJ+yko{Z_Xw)H=6SC9 zaERMDA^)5Y3@Gz;&blr$LS#WX>pEo|xAkxbOFGd4#AQoZ%-T+In;I3qH!7nnyhZ7; zN8?!djfp9RyHV;kf-{6x=lY$(n-s3i{{{;GE0pfPAX~S{VDUW`2ERqGWr8VpVOADV zs|oAwBT|=^{iqpI&k*~WlOe-Tonae8w8wyydxr}ca@2F-h{=%Ma4c~@>8OdEx7EDE zb_kzAb1$nY4hjQOV(V7}&wf4OrDFHRhiRV z_jZlg>-JWNqFnMcs4q|4s}mk-sqI>|cp3!~!ozR(fLvy`1@fwNyL?kAoOQiNdlDcLXaG6&PE?G+5uJSA2ji|#}q5RCwf+rTT;B~ zr}WsPRvqTd`5NBhSH*p+M6%|BecOnr(ldEW6Ysb;#cPZ-xW|K+T2jD?{Ve zmOx?kxIqG>5BOdT6K>i3Kj{7c`gQq=kS?OG*47Yc%OmA|9Hs7-Fuh1LT6EAB<2|9a zx)6h|4Nx+vSq9Ijj(3Hl-NRuv{M84w&GCzg?))U1XOBs9wfvU`F8x$YxX=2;NE3K5 zuKq_>{{P^s+NmR8XIG9u|Hsbmv~K~Se1@h@mEgEXiOp-Vbyj(vBt4mFE5a)Fu1%j_ z@X%+37D>9eI_(g?p?2}xmO`{_=j(lBuVOD3=tmL4V}N=(u(Fq|lMi;>4HKh>ZhTj! zZ}sgQiFKmKULfa7!@W9R+-9g$?SbSCQFWq>LB<82hmxqL(922K?mI`CBPBoN3(nme zz$zRiA5Q5yyyCsqW>^q7I+J{GXKhb^xNcBt45jxJxsQ?RI{sv?Ht@+#@b9_gq|=O) zN{|IFhvdVf0YsRhOo2c)1aQu5JD3KTKZ_@0%X~9;SrOkxOU;YK{)x=nv_p9uy_Qgj zH^~SZ4l8~~ks_r1sm?;&8)qC`)MM*oez-RXAef85&DDHPWmplp+B z1Y_{)RPaW>&A+mXIcu(GP4m$tYW$ih=a(Olbb3Y5n|mB3FM*f%R>w9)E%z^oFYW_& zX#4<_%fBZ~9u6;5Nxd!jIG7z%bJ_KrTo_dGsiNX_t_4v>fS=UKic zSpi_PLLcIRCgT}apXfl6d+h1hTJ@~7U@gYNd}T|r%h_DBqg(9i+L z$O06sDjJRj4&M2%Og^5cX?F^WO>iwKyiOmlE^wCNh?fZP=cz)j%Wob|bs9Ni}(I$3@4TUVQ_UYzDMb)-r`8k8XbnI(6JZw7b=AiTUE!J|0?E zq2v)N@w2OP>rG#!Q*F6*#K;r8$DU_WgQaXafexGB;<@)Evc|ao9naXExaPeE6^3I= zW$^6Y{{?voJvia{fKq1%I^7KGv{ey1qVt$08e+;UN%zsFUyHBp<+v$Lo-E~WoL6~w zc%RmY17AyX0A)!a5yU~&*(vnkrsk0~0S&SbdQP5V-_!a9kkyEKwzKGn21s>m{|8dKTG_sn8O39>jaD?WWl~vC1 zn92=1+37xduf4XMjh9^;ulo-CBdYhnCB$*fq4$ws=YJI2 z3Lm>?EKOn&L9wpoPJga1);WXdp+5p!wjD%oHov2C-4lVqB7@PtBg-j2u}P!ARVmIV znY2A)&?rZ=%YFWIl?;E*9cuJpp4R%I#l(%k9zW}NORYRg`+mWf$+Flv@kcvb3yn$4 zvf6p0(!A$3)%j)dQWRbKH(DwMJ zd&QN3mYU_KWs&bb(Cantt(i^p)KRt1gG{)vHs?9P^l>Lkn=4-<`-2ilj`Ls?@HiF0 z4q)Dp$r5;nUUI`fINrSnDzq=Ldv7WIa|o!Y)3!>5z^lpa2tumKa1~vWW9G8it9aX* z@lkUv1p9X4Xc(J|Pb!r<$LD{+=opax@4mznh930K}peNyrr&W_WgzV`dsUF1t|=J+4jD=D+NQBTw69O;Q~zDpZE z&`$`~XrsAua-Y|v)UN{$Rk@$zNkKZA_HW8{@*wQq9F=hdE!ef55Qperbpb@r$lrWx z9tp9QoK`Rr;+BUP`#885rt>@KOy8e;h$wIB3D6jS<*00l;&&0CKnNMJ<_L(iddhhM zZ`GNC42LA{ckG|{wptu`bY&6+1B*285Qf~aDX`hcp7WG9)4=(-5Xf--)#3($XF2Ur zG*%wBVII9}8?yD{bS3V+(EM8i|Qh%e-Ib2SwxP<+n!2;?X<+&(h`$$ZR0nYVM=2ptP7R=1PUS z|GHp4mObd#zL)uO$tCpW$3I*Pe7W>|p1m5;h2pD5V~Sz%TiY@IwgK%I8nZKJME|DbVj4lT49(rzaO%RTG_fnjp{#9i|`Sk8<1%Pr`t&9|b&O zCbAWdwZxpX30+k}CSs(J#h)M9D+TA+e%9H0|Lj{?oI8U$ zC-Up7Z$g=5S#^Gm?6vX2*e~YOE6G9KwZE!Eg<^SsYac=T@Q9ry{n#Av12-$DM4qW- zzlNuo!5T4q&yy2M-&wN#M2>r$yjb-;kux6F+|vO$fxDgm)LtSZ^oJzx^X#Y()T~sh z$3!c538YSQs%Ydr$DC8q0roepvKnaB61dt4E#8Q)8D}66wCqrFCtl#KJ5s1sN%c?4 z=f@2$Nz$L478Yb78uS4th0h|2iRJj})NvtBgbR;imO$YO-n%(-7;)U%Oh-L#F$y*>`Y%$i?4Sb93>H3q*d1KXG@@dbQ( zk`OQS#^wC!W%xcwk365iw`|;esfgUKHo&eL?3L6PQnnK&<;K*v?Tw+*c zsH4njBc*C!IlNN~3>_Rdf(ln72^aY;O?AnU%(Z`q(JC8RY*$vS>eHK%?$l`><<$vz zW*HI4+*6TRQ_o*1uz!IbvP#Ofwx@0y(fGSI013Nd+h@y`3p#AS8nxfZeDA0aP&(@h z&%&JARqnQ%#Rl*4>07IAbVs=8i9;BRc;kU(n^bV!^~8sf!0Je_=SINziw+rIoySyv z5St*XdP`1+_z5g?t7ZK@`rgr_XuCDx5=@;6vVehq#oSc*=#2;1=jAZYUBoHu(ls;h zCcgahW%(MJkrNOPiRL)3WhJWUOGU%GJd8iCtZx=RseIiWs~s2fqke&zI8sR&gq8az*N>gUTx!iE4 zSCo7kG?NDgF7alH=!(%Q_Pz)k$T(v?=a=J=MZQkBlJ%*}shAF)% znS7Wsi-~%Yu8nD+oR^4r_--)A{fFXgK#W#}q8!xn+bw2x&6+<;BiZ-wek(1DyQt7t!?ZpRx$up)%EcS{!(;>) zLj7-NE~*18L8zI#uxLK*<>q}{MLN1vhx6%ZW8T@XkQoTA0Y0}`sVtTNZQ6Di4{B!7 z@nM37{P6tM_&vr`FaUv=6KKToJR**mgsRAYoz~o!l5-s$*HO`rYI)7{Rm)H7{sZz= zqUk&WCtz>s7`$uFDOer0Xdb?%BN)oeD}C_6o8V6`+()0`H?FjoD-c=Y%O&;9LRxO2 zntge~q{l(z#u9IQsXEt6k90gHO>N>>?gd!E=_X)fJCY6pP~}{=sO8@QsCQl5X2tHD zo`j1;oSG@?xss6CS`B($yL!O+{vDq;nE_4lPk@dlV8V9~U^Zz2Mo7bX(j8(>Y0D2m z9bwyF+L93qU;Pap^yswrw&Rdhe?c%`K+Ezhwuo$wpJ;aR>r%1{nk|b$x-=o1%lhl) zMm?T}dCNk6(HovQH4g)1S7SM$?dVh;pu8GvfGe~f#1~59Kvd;SrOg#6t?rrT=Qqom zx1FE7;IXY>mG1N^O35yL2`0U!COGU~V7PRDSe`K@d?HCoB4TM#bkvFd#Thz9EU0+c zIG!MeG-3?=O*!D`aFKkr8DtsL;j(pVlv6%NvcKT8}WSLe2&oi4F0h_g7_9(9t7%Ev) zKJ|0e0lSf1Zjm>pdug&w2!fVKUPeE63`xI?BPLxe&0a1aPYc>%euLoTe-)W^+xQGC zOI=nGePE0_vZ!=dDKt>`%7D5aOCM+8;}75ZhKk5=QesbP_`l*#FxU^E_O=A=xbh%`d*c7PR)BMrpr8RD=`p7^4H;ix`z)2 z3Aq3tatQ{_ixi4iWLc(awqO~JvzCyUATQdzfj4O8 zu1g}AG%gtoT{*8{_J%$F^u(K3oeOzc)G-?Lcts%^Y6I}gMDwor2@fHmLAyTn@zUnu z;@9nh)%9VZpHrdn7>xl{NJbR07p%>S$RxR)Lqr5CP;zWLNmROB37+S`p@{WtY zCIH4jIE0Xz<~Caj0EbH*=HZ(2FGc+wx=-M=8KAXfSV z+ApjSw--|W0cIB+e21W2aJn`lhD|V&B*>|jO6W%6)CDr&#na6`y>BVZ6*a?>AA>19 z=;Drkr#Mt*Ak{?RK<%A6m!nW?cf4`kdIa<42!gqx*;)~s1-5O1pR8Y>r7(Uox|ENj zDe>pMiAM^tLX(;{4!kbf3s$ZJbziYPpoQ%epPCHuv(op)^>05~HSTvqJh-(sVDpOZ z$`10{auX?RPS3(FF~~|e@i;eNOC@d3P29f9$s+i|+vXWgB9>e0VWh4yFPMNO+Gnz= z7MJ5zecMWd`I6w?%stw#OLc<82v4mB@46Z!rGJm}Zh+*32d_Vs+J_5O+}qRRr}zy5 z)x;hRr`2_)UTP<;sS8NqHk9W#7CsL^)=z&?1Qq1G5uBYsq*8A*`8N8aXV~~iudb^H z(_Z^GT4N)zhw<@=^e{7GUG4s!ROG~5dZwV}rK-CbWr{SN>~v`A17NYW15{w13LEDM zA;v^6^d|>P{<3nCMHk#5Zf@4&^W6jn4x!YDr4>tYfd1DB)RYX+Y;h7e>rWn)WjW8M z(cQ=wQhM1pI)jnlk|joWne;rQ+OM5%88Pq&EhN$UCl+nCT8dC#I~U)(V*dOw^mT@- zqMP(RNt8^1zu#wF$|{M%8Qg;UqHs#3@mlHz$u({It}Euw0pJA|j#B(2S8>ex1=iRJ z^MZ}I{7%)Xbz@D%aT8H6Pi>#!;M|^|7ATa4G;fyH;J!Ck7qxn%%fc;y)53L1zW|Rr z?ga7s^iBLm>E~E0)5YZ9sQQUwe~1ug8{GR)2H2#0vvw!e^!#YxZakf@MbA?l28ojz zN4MpP$;B@J@ek;bd~f5y^{WNw(VF4m#r6WUJ6KMKaqpCF>yEIg%jTCE+j;61DR?R< zUyt}q$6?64FiZ5<&3eJ+nKso-=Mr+(kM}l*F6!jl5a((9pdUEjf7YU$5TJwdSMrI>og*1;U$T=-Z6~ zuAHEp0EErf-V3ht!TPRnZIedacj1@QoEvHP!*Wu8zvY$coT^&|)ULr^EV1|t`FfyE zNgr~?hjg^%H5l3P_RWcF3pQzuwcqP>jw-;mE+tD|Y#m$BKYmi8Tp4F{Dcr7Ylw{aH zHzl8_Ft4NHJ0s5LGB<`S6n+-=@Ik^<2u-v#Jk@AFWIc&Tw80nV{XMZVEm%`VA>E#O z>_3Dd9y)$={#p^9WHIOlKQmztT63Rx$+vWDS_Gw?5L9t5qdW?UKThqhi199W3jp&g~Th|D=51wfTWu#;!=%?!E*Wh#} zLzog)SuN6a)HUO<+xD5?Sx#iL!fe_AKCAQE{U=n?fKchP*?F_%{ZspJ3!*xJ^DRN@p6vOZNb>F))O-&J@4Q)@Ha=&|7xp(Znl{c^!l=J`sEn_YIl0=SWM4l1xb!-0a>hA%N6H_^4+0hkacjD%PNb|j)-jeGS`gFv)JD^_g2m-P1jmD}nv@``2k%>sw41eQkkAi0?jNujr#F0|Km@6dwD-hu)GU zhR>B!+P^ylFN6w>Rsa5@4{WwJ;4pw(D0Q%f|5>fo(-VmbSBWy}5!Mep^NBO(7Oc*# zI-!N&m-^_Z{OF0bU^QypC*)Al)y8r{!aTEH2oDH8sPv_pQrzeU>6_?=>ipufOI1o= zM5))aXS{t^*ZeqdN*~3l*yORkX_s#jWbk`Z#vHq<2#dugEmN=ej`hx4me2RHOxgPh zm6ysgzN!rNaocJP5@U#s$JAWMW|BslaYd;wr`ZZqg;*_Q?sd^alA^g_ud!v-eJ4iV zPzGR}@{S=M-j*L`NeEXjcdiPrKOEj~XbSskTw_xgYsc$U|HgGGXnRpju~la^a8i0H zyQa>&NtT=b@4$@B#LTUM(`Fz)%tDSai@&#+6Rt>OJBY;%VKwYJ`e&p;C#ybI@iQ zIX8?oG_r}ci;?-(K<_*gwyeh7x`ld(_il|?c)k}I#5Hp_8&iJ*WOeEP1-r-4d;B_M z-E0ZLV?yNx%GavsFuq>jw3hVFN<4IQb1?d4I#iSd}>?e%kh;HSM@E!?cWgaNl?kL5r7P&uenLd_9DlUy zXM5Q@7+YjH`@R^jov`8a%hp^W8~|VNoUY4wTgipuF+uNU{chq?J4)&zQ#5aAGX`Gs;}T ztApAo*@<-VnLC>lDlpv&k2&(gZUL(`Y-kP7Y%gm&P5Sx%{8BDK!=f5OuozUBdZopB z0(BWLQqc8ag7`pN=c&{3l)9po%js7;_h|X-v=Jdc4wt(EN_HCiWAWc4ie^Ht}-mENY7d@HuiOMfO34OpOQPuBy-FfF#ob2IrU|Fay%_w$C}lV8|F6IQ~6|?8bU}7phYuL2GrFIf*f=GgzTGX#X_kEWdeu`?XNu@Dir398w~ zQ-Xt%?k6vxrCHpV*oq>e>qo91-=}(koy!ie{TUPY+YsZp#sNK*QPmPX8|$sRv~=C? zTPNLGl}-4-H%9++L$!{G;D)MmS@PolM9=13Ebfz=K&p+UR%^X{t>O5BNW zjZ)|JkmMn+USZDp zQ5STHpcp-v&zy`cA!_S;r`|E(uOAuws4CFH&--u!@p*X!e}W?TJhqD)=oG-E z{i9O|Ke$H?o4rS*rUXffQph7l8TQF`AtgU{j#ye5d4xq0InO;0(AOcaY5t#OPVDC? zbB8?+j-+iZ+HB^Mc-_m5AF1MWc3Mycz69AB-KodQMDEpBGSV`iFeh1D_x)u814m+@ z->_(6H~8$7R)IYRpk(St!c$79_8sf7JSh2}yF4W5i51vr(t3ba0KzTk537e^e*Jt! zk5`ZQ*xD>?jDw;pk2&(1L@&$l+d|lq=A@tw{KDVZ&Gyoe=;xe@VQ!PuXXZ))zMZ46 zLH|Z0w_`NQ8FDN^pY(igUOk4~>E_W7kr*A16`X<43-3Uf;E{{M!}enml3(a$_%fKa#uMm`?n95l_YB2N(7h+Q^tb@W06>?M8va#YoN%W~cl^DDZAX&_$gKk->y_7DGIO`h0>*Yw6H+T(nLtKD#p#zW>HF9bmdLS!s z8a>j8ULHd%DnWohSIpruPd{UhIvCeUUR=esn@}Ffia0FY^$y<@q^a z5jsu1K)CjCWQsM(q-t9`@o=i+V$>@>Q`2n2rfJ??F5aJx>$>}7B#Y}STZsY7u6I8A%%&~oDa8D+2eWPwvFj1n*SIMo#e7r0m5 zBQG|Y4n7;)HCMc{{>cK%^qK3CF90j5Ms#B)5e>?fiSa<2g7`96awOWMBo%8-TlgVo zJnK3GjdOFB$oq7Ecj4rLLdQ8cIUd6XOaVczaH|x8TFWEr6WK22d2V@%EfU7=$+`{8-u)S)x%yBt$ z9TFsrlDEW?!1`);$4?J{x7W$Zfa2_df}m$)M^&m>39)kcJE-<3{+l(^GMCF4jl&V} z&HsY3*?%xZHj<~2!sRSBzB11DhDK(Ck@Y^+puk|A&tlyZ%$!|)yh>0Fy~o8zY4Zke z=$5cDfrlnlC_iuu73O*mw_sf+OBN8{HN0pm=1l2|JdXgCgSBQgXClJXN;~_57EkL> zt9m>7blJgJOB|fpuNC+s7sA##6euBlzSZiqULI`*9G=joDr6)sI-kQ>R@;(MILP*$ z)mfXSvnTd?yi{caqwiy*z>38k@Tq|CXSEF!vE2UPH)-r7o(A`n0^~suK@dxacOH0R z8+CyTAq%y;J?awD4vJF9JS1^M8w^y+SEUZ4(Dl8=ea~j!Z1x!AT)kPh!;6D&!AdtXqUjOEfRUx#(csj-oaT zGl`$~oFCsQD;~Qu(9F5vQqdqZAxE{h``=_EYJLtkhR_`*-}L?LkbVK9Usf5@A!#M|3Xy1TN9SYEwEy@WmC*X0clCO>$R|6GZD_|a5bG-}e^3-TjZOqX8RU(meYTE3Prf^+Vz_^SPx5J4OzB12SP}U>|_%9v8Io)p#3)k*De37|Xv2kWVo2zK` z#^>Hrh%cxtoni-hkw&EQUyu{=WQ6z}JOC){1c-N*ZVvNn$K4PPwx{gKL~NkkUgiq! z((EV-Z9g1YF0V&Sq}`mQ55D4<3mBUXrvdGj_p|Sr;@2S=P%cmm)hPpIf>uW|2b6N% zvK;R-tCoZx(HxSLEL_A@4{`c_UA(>3^!3l^SO1s{x|c3G!rOgNZ6NP?HVkS$M>0zf zDpT$fR9h0>MVX1)6|=m52P_5tf?TyZ(;P8h=sj{1-?ry6sKPPViF~zfjxkCcTq=KS ze!vj->gmm*7fgh<`GMNGFi4*7=P`#_wP`OT5~qw>F;cKDl<`^>D>0+Q%7yYO6=H-c z8NQZZ(mmHll}t?8E5=n<5ZgYF_c}`?{`^+|?N-;FR&EjNl2`4*BT<^0tl`2het^N9 zmm6r@1pDY$ZMJVaGIpwjvTDopth1@_Uo6JWo(H2=q;kM8e)wM7|uUzZ*(yw%@&^HL;mpjqR)^%CF=8MD?87vQ-F@LlBS6RgPEm;-A z)hh2hZ}NmDGi-$dPWG~G8?67%UzG-qzvNiUNr_}P1u6T99;*wZ-!89gyY6BWa$*+~ z781r5TueVi>8O1Iy=&|4JmNrz{G__Pax8M{?pKoQprVSKTvAm|P~g{TxY>m4>3Thx zOreo@_-RUe`H1l}*G?zGC)B#UI@m2;{)I&5(^vPJ>Nu$%;Y#$5hL>6eSVV@l?v(BG z-_>{@^Op6AR>q>g8nJvtF7HE}I8wjS;qk1Y@|$Ol8T84O?yj`ir{#zGH5lflgo5s; zD@%0SCsPet}WI=OrE6?wQVXN^j z<=9%Bth#_My7A^mpaFvG1Iz#`fxAEKR-(M;m<1=D@q-wRWsMzd$Khe#>{R7f(e>y# z-FI~rrW^eBk_%@%ZVUXO7^#MIO{YP8#ZVeqm(Q2_Dl!l89KTDst*?VGk7~jY$E92c zdz&*0|6@4{I{R(dkm{5puhHKqdKWk}eZhA|kwjy{+=2b{SBh#VHFpQ|}20vt6nI^Je z=HZw9CKUF8j^F;8fPK>X)8?-8zqTA@loHztAp$|D5p{vD{sC0CxI&*8Kx z^3G^yk!F%;M7z+-r(aaUveY^<{Mc4X{zgBOaFh#CA%nPPT>4YW{dXTWz)MQPU+HdM zAM)8_AUlHv`RJjvNY>L9x&(dY%U47Y+7cf1HJ?+> z&-g~mas?q-2V4RAVWQ}ZIFZKJyupT=t%dZWm zX#C_LE4$qnr%b|p4O_3@+pzI@Lu-&pLogg&P2^bbwDv+e{g7$oG9v6YU^~~2=WPk6 zFPbEJJ*xh(5ZAmbJo4GEOJT$q!!5@NNWL(hPl=cN%}?CkO7nj*9RrNMjmJ}UEa@*4fwp#E}sTqk{>j(XrzJa=P^vy1{P|(>B7SqT#fNP%FL$)# zLM^VOJSMW?;)i>&Yrwq>F^3tY>@5m}>pHM*iIldaezFflVM}aXlxIdb2t9|Ze?jJ! z!jkvqOviZ+GNOIXeV21#)5?Ijfks*%3%k43X-`Ur@;`kjE>m1GwtgM&56v;HsV@8e zrrXQ;bdBIlO(C<^9Aaz#F0OiDm(gcKY0ezc397nsdZn=;H%|M?k{sc`M3#ajjJ;@6 z+T^e&Ux;75L?twI0@(2$A}B19=O}^P9u!(p;*g~j{!hxh03+N!xp04QGmD{{Ddmw> zuqXX7}D2cb$BU(HAdMCfBO3!smH%RY4(B(Cy`K;b1{Ea3K%)(nz%m0UFfeH{e z@F6#duU&IjlY>j-RB!xJ>)uPE8hEtTbv94NUqeEJKJ$7nCz#Xv3=2Y3zjiSIDNn#M z?LPCH;26;(Flr8t&)u};PmIs!snZbB6)UzLdUGH1{9xQ4cDoc8l8bYc^gImTehnOG6g$B@;eJqjL5$+;)<^{gGyB0Wu}dKQ zX4pCiBfD<=0BR(EYCQHv2J{+H0iSZ0X!{A?g!H(U>4R$8;4qX*TYwggIWGgoFY86C>c*U?&7YQk@rI+rsuu{$7PQ#uZNi|GK zl#V#cSMHih^nP%Jj*){*m8H+OEAGsvSFTv@f}HAsXtJ`WrRjq9M~y@et4ysg7cp>{ z-`GCba%V~^3JOKS#M9KZWU{QZKPp)94B1eFfi&g+S8PvSW#n_!t|)ug=AJus1K(am zF=c*jO!e-NMdq~MW8h*_Y>fTfhQHa?RbCvJ+afzoo?eBjYZp<+ROU%LK!O-(+%(dOMW~$blNhoSqD39mT8W@B!PHoVQTtb zyln1ajBI`TOe!-$VxV0GE78*{lv*s^ezDeL?>?@olsK4eZa%5YS3Bv!JgJ#~qa%+d ziuw%Emd5uI^`_Fly)CjTdYNOM@fvmgyAM6S*Su}Hq$xJ4+Q0|(KBzMoyD<8r6~@{1 zC7s%#K1Q|oz3hR>klQ`_SQHoW{3o*9(4SrDFWq0C%W5rT7jVQCX=)22nOt{K!5=~OXH~O78K%`Y@=r7OR-BfxVw2!G*~9x;!*k=SMqE3CkAhA zk=#0`>-DIK*>l2SeBk4^-b|Y4R(};c3RhkeZpZKiFt_7}QB*A(@KUqJq4+~aQmw2p zWvsUPf@t+)F)=2-*GBK{RO0l#-8O;ix;%+x+JX<=Y)m%?FJiQL4+1F^$5XTxb%}a4 zrNHiPXRKu#EExwc|xtc+;#1RrOOmFx4u?k?mdhF17@QC z72y37`(Om>n@P=eCx7mkI~a0p76`xQYc(4 zeIa%@-H^C|6oyDo)qHduKR0I+SB-s}C@&lnZvFB`#12^#;^+fK+B2`EP8Op< zxIJVR;WAM|=y{@@e&PL(#!EE3`%6(J|7iSa*=7VvMO$EJiOf4lg+?I8cJ_`|NkVdW zG(3D=A|6n)^o1Ue#Q@P)Mx7|SuNxc{*>U9O{^02Ku|~^)SpKzqeI6F|skqGdDb3a^ zo3~R2iZ_G>UmcVq_gb~g4r(ZU@N@ocWZ~(9^N7DERBx_|FBy!v&b5WAIOaVchlO2? zI@A^LYOF$SG9bw$2xlGN?Z|1FP2tG!&GnUO*JlrL{ z@pqxqbd6G1Z4D#RQ@9!DJdT)f2j`K;&V0-0v81`E>iJ0e=F4->!v1V)Z!MT7+>ZQY zN)v(*^Uvak-@8uZ92-aVXUC+W3b%2Nz`p&Uvf}$sp}`Fsm#3v+1N4uQzW;HP@XMxH zh!+us2)(UU<;@mS+5|STcTQ9WnBFRe*J6Gf8?$PCzn2fnxwEFu0jj0~Z}=q%^v$NF zn`cJR4rcJ%oZ{!cmPow)5r{!=KyCi!$lc zf5Y~PnL+T5k!Xh90@sKJ4i1HB?JgLcX`fvzx#go{>TITAlX@auQ}%$bIdWCxx;E#L(!par(D$* z<)f0gFRJl!guzAzmif?-fCa;WDi{%gcY5|A17t~M! zEyd)`hY)}6?L~BG`y3Z8-UW$^Ydw=1Zn<%*RHJxABC+}?*GD}2?G&s6bh<)cn(4PzFl%Gla*kgJf#*D%T z)8uNrsP1mA28{XMv(C0Xy8cB`B1e;*Lb^dWYoa8X6}}rUgT&)n@8E$#;6)q*hUYeM z>t!-BR)+Ra;2$+!LFh$L$Uj=-X4+)ii*mY{#XZS3EW`OL^KM^^>2E%WlqCj>eM*<*b(f(Go3fXqig{5Neibj(>?aAtJ6?SY;uh!0fY|MM zdB`hqRrq6!((-YJ4`wf3w>=yVUbRRZsFQ$ zLs;5-r9Nfe48u@F%Nra32=6uwf>STRO5Y%dk~Nh%V9oIqvpB)UFHzYJ`l|fDziSM? zuhAP6>@MfR&Bq+KdzS;hwm&Jtuov9Z78sLnPRH2kUifx+4oPJdR6i68D*Ze z3LxwR352Vn1()QLK66v`vsheQ%jCH{gP#@Lb%{u4oMS};+0bV_N&S}g95m~<3Y$FQ z9oFr#FMM{g$ETtMk;a_! zd_`VIMF@Hu4}|YA@O=z*ofUA+1hK>xyX(hc=H5w?T#6-&;m4UKj6ZhlK9!^uN+n<|ekw0%d(0`?<;3*nz5(;+!ER|LAZ> zH~p~%QgXF)tETz0Hp#gfGE zvO!}0JC5A#nf;p=+`5Uq-8$T2VpfA>PX!kk&XoUj!#;gD_yHIR%U-+OrSt7s^tfr$ zU*)=t06A)R!FD;fX)=7J`f7PInlb)8ty|*Yvlz%L&ObekK0uGrZ9y{NTH8*|>lQz# zj?xK@^_Y5k8PgVYEfOq<9-}zj$&ZnMz7ocO5g>#)#-9N(6u^CeeFFc^i~i7Rs0hXR ze6};(?-c%(%cu48e73JWlB;?)=FHISf528a@=P6Xi*X6V!>6F(BFiJfJ6F|`9#MHn zY;UrE+`Jn+1gOC45VBuPE3oqMU`Yfgj(Kig`AG`4W8I9LWk;O4Fz-pG_<=28X@$>Y z!*OZUZTd=)^j2zA7D+AhjzNJNQ$Y)d3!%G$yD)Nx9VAXybEbfU$_uO-8x)|n8vSL^2 z0MNTUIrc-dFDX>a3?U=;Jt5pxW4)_Iz3 ze(r0b;=+;g#QNtV!)~#Ye5XY7@x}ifEdS}_uM+rtqhXhD>?Y$*3!1B?Rbsi$8V1gx zf2bM{1$HiBBtNFXHE8K|e!g%D-s$rsI^JPAklt>`AliYuwe@b;(uemIq*DD}Or38N ziw(l7B1xKRa zN7lq&*tJUUWyK@RI9`IgZSRy7wSRXScnFM7!Y{4( zOjyv)$C}QULuBnJ^yfxC#1PU0_%wlj?9$de!rCRA>nhr8x8yr6Wx}YUPuqpF?3S&J z+#CAbAUVUIPxQ!|18?9+F&9SKOkNwv3OY!Awy`VDPLVWb%m@vCuCkW6Ws10@2pFf( z$aHFKNgk(Ir*2&|7WlKX+=}{zy>1{Nz=B=DA}c}o{%-rYoQB z*^}6jmlikDM4W6E3!W0Ao~O}2u0)V<{n~|<{4Q$CVy5nM)xx4DdiWD=-26}yq^pvC z#tb9F*%)(a!sT-0zg`bQlo232R^gZJVoL`Rs_-9x7>}Tqer&s!Row<+93!EUwL1T#o$R)Zx8D zS~fzgAo$xI43N*a*J9Haz!bFk7F(Ggn^Vm5AF<-2VNdnslAfwN2T+{6A+QlQDIGp2 z1oI-lP@n~=%!sfEFb!Qc^WRpvQDLUkV_*3(xbN%TwJ#d_ZrhIA4*RfR7^A%Sa^&9O z4zc<}n$?MuJ)6Yu1)ruGpI?UzxYGD-j8_$bxjtsXlvLFvjNpS>J1gd4`zSW8C)>8h z);7w+e%G9|ScWD08PA9jUib-y|qafZV0m z@UExdMdu)knyjh*0<~a^BYT>%0l7F);i?r$H(oT1_fU?vJiK zE@{1<9yM~%^YG4w_H3Shz>1XBuqnkkQ`a}%-*92ne<^2W&EWi0p=+jwYk8~@w3uH@ zw*Z_*cpZf!0;N2INlHqk$e<%SV+0c2j(xoEY#CS`8(#+B+Q zVC@|JG%_WbFb0pna&gYI`bviRzk9VVg=IczjZrtQY z+HzZUP=JNC35<$pJ$lC7#vRReITX%hyQG<}J#BiVR-&PDaoK;DBXK{r{=gi_G*H{- zA%==xfrcumV!^m>9bBs-^K5ZEx=8p+Qb5{a!Pn2xl0pMJl#py1O#x!2lkA0pAJ6X* z1B3E&Zk!>8QX3b)*!tc6Q7>`3z&ke=QDKIo4u6gqE?^-n)8!t3`+b41R zvyF`aUy@;xu0I5qh1IL5&RSM`Cvi)gPYOLFKk!|v!R3P0`Cx6iy4-grafumk%Q|(@E%_uJJ^yB& z4bBdN4s&nLCfDVNw#zBeyy?c4@y8GRCMdrN6u$zsX-tvRYHfA&+?YVRpV3uywiFx7 zA&6}H`TY^5uf_HJBKGI=p|*?qu4F&1HCzD2W8oc= z&=bQU?h>cMJ&d@1tr4H2-C+%{-`%9>em?6bnL!Cf!P{V>8+Wh%7Gm|;PZav1DB)VU zXTPBC@Mqt{I!f*9vsGfIcT2(*bFZ^X5S_GR%*U1nmu#bRYQr_GWIwhATj}Q(X!PZ0 zwgDma!a;JA8FO5LMpape@mfH2NGCnB?Em^5nvQbnHI0HxF4n!zez&tw^H%%nx0_+_yjQRT zG>Y%`Ndv#k8&AWgx*ZWt)cxfd@{f5uojtnbxs(gRGRljB^TinG1q9I{FcL%usHT3g zQvj57K2o=JAD*jNHsy&)mXX2~>me@p70T?ABj1tpB&d-iN7>Jclv>lN+h?{oGt=^$ zhGpX9h1{~cTR_<}WY~M0RSpd@-?RlamWPw#n&oP}o^xOzriV+{yqu^bu#35@ALA_- zxtO>RGTtk6t7-3zIRhrJ#3yIHmc*a*Kp?$DosG1bGGjmb4(r~Q z`Q#sA?2q{8xT0LVSb3S#-MN6?-WK==ySU+}ZZ&V{f-b5;{Q3RzclaF@)|z!0+U-fo zdag!c)$-zgrAt``Puxs;1-Vw=Nnjh)`g1P*&s{)T`~cD-TgdSEl4~J zyOh6|Yi91o%^ZPL*nNcjjIwal4#%w4c=NoS`jT#8AHo?tWz?lVD|WY}v!n=$vx7na zY%T`sYm@;P9aQySYl@HNV!@!W;UBW?dOw+g8X;FQT`juw%G4KSJ)#8f&`f`?ZKAC| z$_9*vDR@iVU=CiquC%1Kta?;Z@a;k4AOp!Hvj&Y(jC+qV@zCM2qTBx;mPRBDU||(o zPzGFJr^zs%y^ss_ z$ZD!)w9#ot1SMkv{Y;IWLnl_a$HVOG!qa)K~9VKJVJFX0hf? zP@!xt%!)Qw8d2o*bv!dQ(7ZO!1IXHE!!;z!3S^LsEG@|Pbz~Z9vG6eKlnIGK-%!G( z9}PxuGkWTB_$nT>F%zucN)h#MKc?=0d^gGnKA_H#*CPv1g^zok8QCSy^Z^|kR0^p( zZ(=c%cCoKTwM^Ge`nDn?Jm9Ayk8koB6-X1-_(<~!ttHhDc0#=g7s(%G`toqU3<}`? zm(9Q-+9_we7f6TuLMdaF=~L!Z(vkOmAq0dd$&-HAcdrlvHM%WsfWYz7`29+w zf;g0;)mJD!@r{IQA^MzU_^?P$GirIbP*^C(qb6yJDdzUk73F@z(Rzg(htvzNbTvni z0+i*NZZDH=ai#HOxZ4pQXtzpLU2eQOB3!tb6cFWX`2>X1J|7wg(;;-H#!&`i7~zah zQ|uXgNc+Kl;@KU3&@bm%o@@O613ea&Z^l2C^?4(u>PA9ovpt*a%JYTb#CAW>kkO3_ zWJfC$#ipQgMXKC9;iILV22UR))~Arz%F~?N5ihX)u_7;W79w2o^_Q#W4^cZ1?*L_y{Ko6V zRiFPT{e+heBi8@8MmObx!p?Do0EIsrR=k#X=xtCoF>mrBb55WAx-S(AmDe&O(T0TK z0D<@Xn*^t*X?^LBpQ4iNb{rj5+F~7iog`trJhn1}8zx*t=edWXOupYc@7E4HA9ggj zQ8CnqJQXV;q74!g(-Gf_ISqpPevbZoa_i6Z)<3VB8;$%pE>`m!aI@^($aZ5B!Yvg} zmDLKo(vlgp&0c(+tsy-3M*^Qza$()2jgPrYL6V>n_*;2pA2k(E&@hZb?Tmv-&dS+Y1R}+D$68IddeMuh~AUqiFSQN0*BiavEW~5p;?Cu&= ziR(4M42DTKo(NUA(A!}R6PMwaFDG_vfkuRm9dXX*n)AyYjhDwo!%||GIIj`5zrCQY zn9^9G$%m}gQh>|<)KFJdQkK4p^)1?YagVRGr}-%(RnniaJ_eibo2Y$Cz;r4GcQ-qI z6|rcGyG)3nCI-@JfQ-ksE>ZOO!T8~Pg}roTZCK2QF6R%)FjR&L7Nso`}X$-wrF zp!^AK{WU>3xLDAQ)LA0aIi?vicmlUEzxk6o1MT3zcXGjt?;-* zb{~g$lr9!LwtGEr^fH8=kAyxuIB_D#pURK$0gYHFPk&cvzr5_ns}(JbD&IlDHMs?P zKGkeAlVt)M=bcM`L9SuYC_=*y;xNImuLe0R$^n-J$Lg4RerSXjJM3!5#MLSGm%OG< zEbKBo&)s4ynSLYCJ?2obJOkkLfVcwi3r`|bEH98*5^Ph#(!$P}D-ONfQPlWLIdDZZ zZrGDAxz|=qTOybB8e;D@geJpP@O$GMTiL4XSNkGPB1Pl{Z_{d_`Ywn9H=R#I9z63Vi)@rcOl5 zAX@aChK-Cs)=7~3Wi&fIKiTSIE$_oT!jp8z;(5=e9|_&49?75SH%9P+JDoWpy3Ri# zCT=0t$B`ce4|SK2EiO>FmZ|hu?5~R7#RhT(YEKhB4e0HXFm(?;2p#!0n2Bh;fm0%c zD)NAg`2(Q$wGhBVv<96EbJo8gRaxgZQyL7) zkdoC^HsR8W!KGQd_$g6P+yCr7XJTh&H>H~vBj zZx?>X7+`+{8H@pxLQOmjZ1sYybLL+s4Y02| z5`zqOBA<#Xz#gT~%^{1kn|QtuM}D>bE@INOIq4zqmv6}XF>Qs868CuqRo-_m&kT`8 z6S`3Q#^C6>HLwiW9mv|PKE=y;CiGxygU%~;ZBQm{F!UNDIgQ%#znXoA40?=51F2qC zUq|J7=JZump8fcP5EbfQQ)<6;lw~YvWR%`PT-Z){|5kyW2P&|0;9c)(ooZ`i_Fj8A z;$C`!OB_#4rsoZz?3$ek&VbP9LdFc+mBQJeZ65@UyavnfNft0r_m`0-5^dIpkjrpb z0@Wz@@sV03eA=}V5C^!={5ZW~|L&NQZEGN!JYO zj)>{mBvF+&@6{kBDZzEvpXjIqBICd6s`L1pq9Iv_`cMM3uo5o%Xg;`+7%dvMb40mE z;t*BTEAA%_V~)X-MV5a!t0a9bH4f+uvKhqbkP~t280bsp|Cbv&ED-LA1_R`J z>E54~Upc+zQbR0tM1*-FsJ3PWwh zWS!pG8ZHpfC>fT&vG`S3=svToAKS4lXqKyBS|mM;Nix|Ob{N*OE(~oO#PYErvo9D zPjb+WY$W>$MC!(W9<9%`R%xTr|gNgkzTYSPEIEx(ZC=rqs zytB|ffq;UN-3gwHZ6&-zgLV^)8n3W`q}0vcNa+aBPH5TKb=~u*ruA8XsL|YlumH)& zW%HuH`OKbx&%E}B&x{NN7W8;lh0)>mZ;mOLx)dXQqn?gSiz-%p!6Qk!---(sQ=0}U z;fmwjh9fHtji|{|Ru^OTE)v>ysqYI3uA1E)${jk&3I@@w<9M^hd8jT2+iuO5jS-Ey zpCZNjQZF`%6mDi_VJMrYVppV6xNiIuasKi=v7KrAk*^Kw$UKeEy| zChAo9a+ycsn4bKnrVOhWf$CNzIb{|1;0*Q+|Tqb+c#bn!>Hezy;&*y)=g@X zZOrTKm95Znj7d4fuoF=^kB^(<1|rOU;A|2LvoIvWJYLj|An>c&cp=NDXHAlLYb&gc zrMSNOTmTk8N@>~9f$ygwx+k~0P+L`m`SfkX;#)i?2rBWki1`W7GVw3!43@tQZLLDT z9dqkkoLIQt)gPP9+1`r?@aeA`PlwxOx5^=6ScXcn?ZOiF@baeAAkX1d=H2u*c|A0R3;O(F5Gi$PEWn|)oa z7Wx_l%O`ktzytjtdJcRT<6{YQaBjt$qwJ3`{&qgjXTL|5M}aniQIM~peh-aD%<*!J zL23&s3`S8XZ|V4IagPC*m})EAhWy%l{$8y|jejco(oMGzUy{&q04quUH>=q#i|ty{ z{G$3B{&4l^rz2l(uD2U;=6_O0mC zF!0mkDD$cL-r($Tax3~YX-mGuAu2txVb6c_lAq6xoWu_}HAcLdx1-%@=&i*igX$J*3L#w08XWPa1llYY{apedcNG=A7q-)1BInodEb_<$o8P7%u ztSM+BixD>QuVwdwydD}FU^r4RK)nR3QS`}8ptr`T=0(#AFx)AwQ$o+OMyh*PJQ}heAI_jn))r4GPcN@~{_SO5;tyAAJ@uta-1lI4PcfaX zu*vxjW^qaZJ&DmxxQ`46|NrJmbcT+qUVRU_d?6)P@W^&roFCY=r!j9K(Xt7#o;w6nGm4 z>rPyr_bRC@VSoFrOEG7vjv9uw!Tf5>_h8OQkxp?~QN2@b)c0oMoqXtynM*Swh)ZxN zkY}TIJTf*f?@Q*KFI^A6y(tT2@JO-8jxyJ)GsAu%6x)$!lDbtKyO*jBAJ45%JpNQk zfw|-H<6~Lnmz(zJed{`t90}}>j*+`ds_8eZ=p-IQe!ZnTM?!Vxv6CfPd7j?L)9bS;hwRaHsZxu_SD&Li-(saDC;=OV!))+-=`IMB-IOD^N7&DZ3+jE_Ys+My6lupg0(H8=nhOd>(FmCZl%yQ=aW-;cN z?-brnsu*p#z2DKt_>!zqD$b60X#L+);Proh3hs|QPSK9l+O=Qtvq>`a@v-RZ1G1$4 z>F3(WNWA))gzp@u_mRZA(#8>d%ZqBqu*lj>wI>;&6X$Y6WEpVIbR$JUoXXCM@66^{ zmsSZ$f!6odxHfudiR7s|j2nLAJhR^Hi|?7*FAzIRjVCt7Z_hEuS-;D*n-I%}V3K4r zgSZ+5x7q%eCMXVy%}F;7l<|$SPMo!1-e9rmQazKmB_6U$j@NhWA-dW8HbJF}x8jvC zW^chT3jY{$@t}>1-O;q#93wLrJzx{LhqfjDvY{^LZ?2{2hMlHs z^46*rXp|J(zC!knuIZ4K?_3j!K#GACAqqh6H939R9O>jg|21O%AaTEh{%ai(L4=m$ zG}-(^Iz;f`&GO6revC_dhdq~?nU*JMHy>n$Xz)1%;!knTiA-n|2^^iI-lK)4pU0Nt_AlkN?VX8_A&_mZrM1m8WUF|v@f3|xjkuXp?T7^>CoBwEw2g# zHo{7XdYsDF>7cGQE-O7huC{0_H%H(Amso1Mf5s_KS;LP^@ANz7UQvw}rlh)V8Bb79 zgsl?G6s}jReq6EB`h~q~s_4}R{FWu5q!bEnF+(J<%JNd_w%M!9iG4S6)wWn&-Wm1Z z;)Cilc^q=RJD7d-;Y=IPISvm%Zq%-UPq+VE!Ti`%Yf1iQUA1dxXm5i0kFDf*7I0@-V)9EcI$9P%;@O8R*j+6N@uqNopWxG=ghcIGe#p)WPBla76p2%(=I%DPIS-g}x`~tH0b# zlc9nSX>ip;yZjCN#I^YjG$p`lb+ymVB=v!*Dc#2~m%x4)MFk>a%DGA@(mKqgLPbN}QXsLmkP?gh zjg_u-^Aozt64uAfWSo)3so1h~V?_yv8+>uVOt1EH89lAv^8~UNNxI8sgUu*#>~F6X zXyX(#!d#vd+AcpK?b=^8HZkC&B^ps|{8e#w4-UnM#>l(R@t{XcpLKfC^Zn@MiTZ4% zmqUVX3eFk`Qq+VbK}eE^D9(j&rV^>#RQ0^zyCrMpn#k298NY9NousS(5FX%kCV{`3 zw8WP_*1|H`QXako%*_Qr?x;7M%48TOi@*!#tS`AJohc5m>q0Gv0fU4}6yD`Qy7VtfIGzu8;A-O&m8$aU~(G2b8 zJhv=hM;o0n5f-;aIg!FDXC z4$FghD^*Ii&7WY&<{t!yl@PQoV&?r!N+KuPp{Y<*uEy&n_YHS-lZkkISkK(|OmQ4PA`iwwEZt=V0Y<3w9$MSn&dRsjj`6L9*cU4jZVQ>K z4Hnh&ZwRy>I$=;RvkPO3!n-0DB-~|scGYQ!u>5YBli&b#diog4m#&M`f~$k)U9(jE zM)i&`inX)&(jR+}D&QlGGXffn7KSx}fN66{5*2YQTbaLfhbL}z_qGdX`2)U|5O%*8 zo#wZdx!D}udliwdB-=oLQ)hc$ju{N8Y6=v2Dv^W07u^W;wE{^E`&Fxi%<G2d1C zviCE$l*zskXSOQc1s<#aUBi21(H#|J(cmayV?rc0yFCM%O9z()t3}k*{ntT|Ey*_n z(;?LI5A!#ENVfm(e_hf}zbw-33-rj*G4ssu06uW`K&NcoGQ)lqsI63h$QCR}pU)lW@;r@^n?vm*EK=8m}XC z%04{rY^(9oD6(==92?eH&Emc)Zt1W#x`0vUe))$Jap1G#24EIioVgiP8;$*REhAOq zpkJ~^@WxGL%DNwG*~kpkqNDFIA^hA0v=8U;j-a39RCM!H8%493IMp)+ztv5WN+I@j zyD3aD4Km&f+_f7&Vx-%6vCpb8Z+e|iw<(0jyyJNQM?$V^fbvDzMq(GRriA04VgTxr zR%Z%wY!v`=PZ@O3nE>l8X`f4|HS7Mxe_>P33{`JO7)@FceYQ;!w&`yvz_c%7N4(`S zDX?@oN?sG`$R-^rdBcm{cc*cjBbBfI>+~;j>hodtPrEW5I&hCp1dF^!z|xL^5nL5v zNiid@>0&Fc%62meLIJHZ|Iz@(-WUd7YVq2*B%-uSqS+P;D^GkamZ~$?KoB}#8f>w{!+WWn{s!O2hN)%E z?g*zWa|@)GYP5?_EC#P3W!CM8dm`FhxG#`n0;@WUrV+8P7SZu-mm6>3T(Y-JkZ)az zcaF6z&AuDmezsicmW4+4x9j&YUeJ_*c=ZRyz@a0eo-VHmY(IVp=T_8*Pah|A-Rjc5 z4T_xL>>qoo52lr&75N?41rol6F+zbu| zjUK7{PWg7rF$k|GT#(}B1@3kj%OFNceZ)(_{XoBTl*jDjM z>Og&T*YjLwPTO(X_I58-C{>~0kdm$N&9lci)Z&SduvMlUPzVDS1JSo=jN_fN^~gRG zMzR>{jlaMj)cllivRHfY<43qcfz7H^LRa)k4O#L4*>kcQf$pL}Vc*lKr5iaIo8o@< z2ul7|*(4g*Co6Q@zLJxHW3^9@vf8(%&psIQmZHL$E8fSam?d6qgns2s98d74%VI09 zO5fZR3^1}vU$Uj^WOOk87}cal#+Ted3JFa2<$e0W9BmU?Fe_y5hIm0b`iAAo)7)62 z;3OTfSB?%fdlBB3WetPmCG%lzFY3!jgtODdR3^6Q3#j5~$Q1-X{lkv}?VmG@X$P29 zVwC(S{a|eOaGFz5x2&BhBm7nqL|Y0foaBeBEI3%hCJZAF=ii*-Q?aGZOe^ztRo>h} zB?%XQE1Zx7=#dHwEOW<~&#F!Bz#o2{IT7LRu1U*{)5RKVuCSd`m!uU76|K4wYqHf4 zf6?d{wLi5kPc^I(;f-eLFrL67M>(aW-sn`ssYGbJGxDb;g&6(YsUlw3 ziw4|9c_)E3g+O*CT9`eyGBs&Rl{BM>o+P#>GziE;CU5No8_$v9T9*q%neE+smjut& zV_Vd)$H@{UvL#b0`mdnopKCy0!{jL;5u?=hW7E5IT2m5#d^>_WP&zS2Q%%ZEe8&33V`y^n));8N!zg+7Dm;linHs8%Y22yB^@2CGY2D1DyUM z=#i~WMN|K8wEfY)!im38Q{!N7*2^t52;4&x@})jFyi7S-sjx`u+TOoLNqnrV1L@>j zlecU}Ejr--69uR7j=w+@G`o2Mcgt~D9n;JT`>_LT4u6Ka-uI%f2|Awk!P7n<^9 zD(7ZkcJzUhsBYb_3ibeMsX6Try%BWnRE{a$3PIV>QBGM8ul%W*TSQ-?_;@@?f*ksv z3|sh0e7J6HR;qr1SM(DF5tH+O7!p8VbC9j7EuMDTXluOoRo=VpG1q3>8?hJEd3pGV zdpWFC1Y19GZ+tWxU^Z$(1ryR_K)3=AgLt zUM1 zgbJN!#yg0RP%9*V!V@IeZr;ck5ri2 z3%fi|OVP+>;T7OH3nAx;Bj<@;4`2Mhd}V? zi<%dLl+^wk?hur2o9ahKz?Hr_UOppqTN3MvG$&W6>5$qRLNRp&P3RlcHoG(-RGEuX zbjQ(~A0gP{%5uz^ht%Nn?wTBHU|5Lnv}@#b9A}f*Rx4JDlS#H zxcT%M=n{56k0;a%06z&k;;}%!z~QaMcoqj)k>qUQLvMO^zx>RFwg=xeT(fVSs4Od> z0}RxH&kQL)+6+_Mw%R-9ck^i4hOBg9Lh&kEF$>G`hkr!mFrfsksu-nhQ7?c~o79-P z;7;D0V*s|kmlcP&ynwT0&Z6@78TFvL{JFiU+_oIFmc7$|N?dBkla-H&bta*<6FzR|1V_#pbV_I-I@37@myDgLQiQFCIP_j1t)3;EWKMGnfII(BC5+j0R@D&$wd9KG@N0u1K}>d8~a z9Ryku1pt3o2#mQXkbgb>V_%AJ{@g|k%Kr6j&xz*f(#ne-CDolQ$idpvP^>VpokYX~ z$UhcI2z66tK0-`2wSRlondkdLpw#f{4V&8Q0}WAn`K*Wio-j%!yI+|27-0!v-FwUz z(c2+Lwqk^XAJ3mW_gZZfR}dUh1OjE&{}CuVy>{^rFt^YshF7n|*)K*O-Pz&7t}el@ z&5=5N`oJppA%lu|s-By`Hhe}1lXJmSyAZC{BBosk_y0@`OellpsYC*t?~r#fwY^BL z!yGbNukGe2966k%-WVB(=D9zom9=#8D$Muy_>R4+NFH+|V_5bayo`T?%23{lgjeP} zs;m?LVXa;C6bq#3hws6}^vI^?FCy&7>k&(+ZR&e!gn`<0sQ}w(%bPWN#_vim%CM46 z0N1o=RQbz58cgh5Sl;?QJx4M6CFA_tL<0JZq<0{6{l2!_1>hc{?gD;>kJ9N?tybJQ z_&c>d>j;N9q*LY^&kG#8Xr?u#@VJ$(*S=_^?jmH2_QP>(a z1TgR_im=ONEe6vb&C*3#=NB5}7JNEziu{}7wND4$dhXn1iESZC=!s|H;c~sD@}jc= z&TD>r#)9ZJ03@Lw4^XF4Y>+dc9{hLFMwP{ci_k?^2KyB{GWZ#If2$evSSR}?5REHX z^4TF>LNdP)ujCv*9$UAa4`9}Hf;?_O8grJ)gQL=GT>Getf$qZlL{7jMP&_@xXVc)Wn8ksDYNcJCpY z91jhGC|WS%QQLU-MFn-(=$K(esd8T;RT+KrQ}ti(sj&Ud<0*s@&^x;)|L_n5nc>K< z#GOiuMJ}jyebBjW0?q6t!C#7`c!$s>r`=Ea&+H>~MvHRJ?ta!>oZOG?w5vUa3uBkVv!>MD-?sM z6IlqIeoW9xNd+7MJJcck=-?a@n1K^)^9~e2SAHzsD7ZQ`*SUJL>sF5nPv{{}^fXYR z|Dd!Ig@X~Q!EMROMLzi7lM?^My8SajL6ybqIM&F5p35n6xNy(Wn(kejrI7Nr5{F@* zK4C01L~S20;QUxg#6<(ZJ%Y1}iXe4yU|{EdS?A=2Yo=b=&{_f++&yARr^F}MYy&n~ z`}z@kWQdlvCM_+y05J(O(+(2`x(x=H`5l%C%^ryLu7aISh+wy(EZiz|@r^w;VI%+1 zD3Y;<*LMFWFG);F^O{15c@>N^a5`&W);ECA3_vCzbOmxzs|jG24CR6M3hoCqcEf6= z_E;YH6+qg?o6dY-V7^ZRC%B~j-ab6T+m58;G01#?A|-Rg6YBng?3`qiC{_Hu(Y@#Y z-sphSii&ZC>&S+cPq~d8vESmmU6^ShVYPha$DP5KhD{ZRsoS9I%ZrFaZ8a+$KF_9! zXhT)#m5jSMdB#%<&&G(cEM*2Hvv-k^>`=H3xD9Ik#QB^($d8hLk#ARvifDgrxL;KM zxYgDAs^RmmEKD5q0Gr}0gYh~b_8CCTEH1+Ldn!>Vt$g>DBeY5(P8F^bAAVwYZWjpJjo1xNYiS_Ty3V5L9a`kP9z} zMg}=Mi@+7Ewmf9R?DG@4HXEaLl^ttl&nUu&KJei@!1{InWXs*XxP$y zBkF2iBZ=W~ztn{mi9l26Uf!_(>~MGQi8!=O#oK0oB^jqO7pysO>(a`CyN=)*EJkVn zX}u-rUI!r0*?x6T1Hxq+Y!AAo%$fK$-$@+`gXzOdN!3D;?lYSfeG9}X9cnOGlri1hXvQ3rZ&Dh81P?9KTrY)b_C*J z4J6_POT+R<${fIyJ%Wo3hoBzzb?*KgkaE#D&x7wZ&70~&bKH?65UGJMwZre>tQX@W zsR0$H)c#s!SDf9mjx$89kryY%4A{>ubEEqafReE`s0SiO2U6n~o;!Q9k->%9Y~u$Y zbz1OMsZQ|{e}dhWU9DQrhF%7H(gEiKfO-FT!bM4cvY`4)0A#vgFSbG8ETwG~cX>Bk zRiA(UMM0T^XsQ+n=XZIzOs*|rzkPc)*PB1n%C~D}RaJev5Q<(@^i9c8gk1cDyD>hx{ z)(^*=$DT=pjo^H603M@duU~=Fh2TIA@@J36p%Nt2IxG!ao?XBUwyi;onUE^;JSc$2#^3jK=l94RXi{pXtfP|v> zI&kFXZV!QR&b?7^&R1|;h=jZoa-iek7j z0t!k9xlTM&f8!HjFn&qlj>L}M%v{>rd(_i9SADkG{YkebG9LyE#?@11`);Yf(Zw(& zq_cd03q@GAAtql8(KT~!9q0hBRlplGM*Kq?g!n=6;)bn*Cx@d5@VJ4hk8IsUH|%n& zx4$Q``hJWqaq%J#Z_-@uvE&eNY!sO3>|q{`gaHLkOc;`)2Tm!08ips?Fs0(7$R&69l{Qwt4IpSiBO$ zw*W5y(yssSOUwTM?sU@6NpHb$)pxHGW7v03h=EUJk!SBL1%djN@CQvhqgl!%lDoIW z+M^aA7d@s~i`amTx99SLOH^YDs7oI+L3}&0(%ayfRGFS103RAVUVM3$t^4KW21cn<@Aqv{uFGPAsL5oH{EQ_@IDaLQ98B_P#RJ9a&A zp+|NZ16|At59FbIyd-xm$?@fhM+@Ouur4}<)38y&@y-8ir1U=(z0@dP?Iq=uoz- z8yU!ieu5psWf_NeIEINbT63x0q!E8j5}dhKOe4#N(_SzJBkIkIB5q(k9`OzH8YKcg^jKn#kTwa4ZN0DarYdh4r(SsKb~cvQof>JH^=pRTy~$?h^Re zj;&hsV5DajqaDT}o%GPKl0J#qX#WnU3K?jB5Fz{ixB~Vg_U{n{0715X{`A2i(M}Zt zT#|{|Ll^02;l4yUwGKJI+-ISo;ZLqaaH{ZNi}pz2>gd$i;$0{B?V@|HJe)%`ukO!D zk?Tl9HZQR@@!7JnZVDdQR`?^gE8Q^Tt$^eNu4vfaj(G2nP4w-0Pi2R6#)vFwwiUL`M_bAw! z?%cElPG65D(u#6(-cqrOI_GG+jNE9s;>GEy+SBXA?Wo1{MnD4_{E`DIiCFV=xWK2Z zOW-z0cupg>bp^W4MCManaBCSygOW8sg9G&^jQl5S)#6^E4_X@`<<+6B-Zl)b(|IZC zqP+gpof#kjf6a!b%fI@BC)H)y3^gB(1==cqLGo1WZ1#sAPz9Gzgl1-Q(WsL}592gH z;~8LVQ>aiE!gTRx@#%I?>xDRNoqa;oQl4o7A#m>qykx4)J0% zT)p2Yis2}&PjWmh@!^lU>`tS*Y`%ELi)%Hn!Np1?`h7Pun|L;*DcI?DC7J5|l!`#y zmmpbaG_1_L3W-~LItG_3*!Vp7D6M!_baq(dx4ZV&i5QBt1Ya32j^WmZKHe@^A}TVj zY$)2ez2V5?7;3dI->xpB{4=b3BguEb)h!f6eD->mCGGj>K)%?c59^=G8qx>Iv(s|! zM@w5j3ggXOIp!O`y*S8)%TXH_s-RTx&=YUwu{O^4qpPFs(IE$`!<{E#*gIco7+ukD zZB2RDv+iU^X!ok$Z`wO*-Ubls2$|*$m^h;4=@RsmW^*3LDu4rE=`W^(%#X|SbkNCb zdu>@RmPIY3-_4jvL(cPF44zFJ9c=Z(^Fb337U?(8P~hA@72@VI*gZ!fkAY#{PQK_uiW1&WTd+@_;1>FE z@2Db7Q4vi4EA22Z7_c{CQz1G@muchu5-13+<8i(i6fHdMCNh znFR18uY!HJPS}qy1h$7A8G9_)?K8p$5CM!z0!Pz%hyCGm9GCX|x`h>|y%vAsxJq=d zJSXLhRa!-WJ=kKjAaYFW@XtpzIZNJ1C-Pd6eI3d%jG9;ou;fp|#ZUx8bc8LEfR&b8AU=GQK3int2YJBgYv}~f)@@Mn=;pC`P zgC+`Z;sTkj4h}gC*~W5_cV4IT@phPpBT3}xQCYixZ&9KZ$&I@*7GxIc4FPg#c=$mW zt~xviUs^nI4dr*SG8%0gEa#KP_Xq-wRC}Pg*iZH_JMf1EpVPfDNA{5zXhyxyrXw%v zX735H8fjqO?keKxWXm_xb_#k!$xGzxL1KusG^5yhy&MOr!>Y7{oaLtZ(VsP5`s$Y1 zb8!x5{Dav(MZ!6$95a&7I`1c>K}!FfT?G1`n&UbnL>hVVQGu06ded2LN5&Tqet%Zy zD9b0(;JWw>&(aylKX8i3o|<$_3%Ol$S5RW}s(2}dvi=M2YXA#985q<}B(+xu?Yoo* z*+3n>zTk2K^1y+|g+IWpbk@dO%zzA+UV&;Zu%0mQK5g~s;%58cJ zI8E~6{MQ8yiM|Z>ZzeZ$S;L$6*4HB;N0XWZsQsvl%7Kj%k(->MkLRN6=zU)Z4pxYa zZ!bWL;=A0#H=ZWpxA`CcK-Rw(f(x8PwFa{PuD@knl}7Wy(F(%acAZFmIcaD|$7D2< zHVQeP><`W=WfxlH>?#^RZ0j&v4vQ<&GZ{S*)Z_m;Y3{^BAP7*L#DdP;-#GQ4Q@#0T zPf>QMBX+E)cQ^(Ps{*RqJX7pTj{A2OE|om>cD|}<=~|gEB>ruWd!K0@%>=hKCu{qy z?KhtQjVYY_W&r!1+`y(j-IsqF1Myc;b@HxjWksO4g)CWnj4rtm49&v+{dM8qQJ1##rP z9@9n{$Ns=Yb0uzUwmoeoYazx_6+p#Nb+n})+O@*O8RKQIsX&#k)IeK;6iYv&We8Wj z*V`siV=CV-kDxGu9=!MB!7BS&s(YcJQ`u0F<@~E&ZPeuRBdA6C8C937I&vD(vGb3a zEY($%V4oR{y5UvQQ69hG9VTi~d22>eY2M#VTT*sddo%{_y-t+jxg-i?C`W$>X5@-O zArb^Uj1nwQ!iWy5WroJ_x1ebqYU^hL&f2J#dZJwfHZn8(ulF^0?m=RuZ$Q5=;g-rz zZg@v{m}Y%m_f=V~>l^^}Z*Wa}ioE{u7sNw~OKzUry=&8b2<5nS@keBG}Er9|hrhANrbFrCwJO#aE;lL~y9bQZj zv}2x~I4n1vH4`qvncW`{$#x)MIPnB<>!fd3@ZL}IO!2C z@9g5K9O&Z$haARW3w=rPGA*gU{D?a>Lia>86K1i!mP-J z-GN~hN;B#qc<1v{Z;M}e6jjhFhiKP!7jV2Vp9KMU@t*K1SIF{%g1HQi)&&tdhZOyW#M_m@ZFOW`YHBylSs;R(L_;4Jt*CdVN2C2Uy*wm; z9v^%iyuZnl;ECb=m`>DbvL83@(vf}z&b+sIEx>SVsa0PyaX~P`MA!4qR<<)g3DcS! zgA&Z?H-aMYHyD)1aB4hK^ksP4X;X+(nZ$5c;$C3|Y@UaocgL|$0?mofxIZMk?Nqt> zaP#tQk6HJv8ffN6I$)w7*pH^*$7g(5%^0u7O*qs#*1R^e?M~nwuNQ-S$-oSLNB?gP z42;pvuB}_(eHMfTALt&Ydg`~+mo^C*y-HgJfXmPBdZvYN8}CKz);BJ4{flqI67WV< zglQQh>i-v>bw~De~x8|6jr?9oru`bI+_D> zKX=HBGwt;-3FB}7C(1SZdOnmX%3c%I5}bbd{k@MxQsvW{Ncoe{PO?(7vvH$0=;53g2a*{2-ncs zAoc&`z8yxm3NJGseG}NCRxNsR&m^Vw@=Zf0^tRK(P88El-szR|lQYW?6h%#JNPODa zWqi^?S3T`qj8WUeNlo`Ejhv@y6z1c@jP$mt74baL(=xYUMJq=Cjfd5@N4~bMaVhEj8p9Z zm6I#pDrUbyS8~rUlNQtK`(jj5J+f~-#UfPb^Ww$lY|)Z?#o`MF!Q^}|?sepZbG;x1 zjpGO-ELoI9!@Y%$<<^NW+Qje1KUex(PM8v9y;_GJ-)@@WOIm0Q^SKrk;N)l}`aRwB z9Xm+KVkwD|`jpKTOHT~Fv-Y);zSvC&`Fidy3QxVP&ysTyyc&Ht=Q@~PIqVNT`@D~4 z#ILp9!9kOAi&&Ps-KWxkCjrs}kn7Ra21cTJc$swQK9a_pMK{%?Jn`r3 z^ihB3^@)=Uq1_L=7QIlOpa+u9;oFJ@Y-ryvJ5eh@am2KSUX)V2BQ)GB{?@Ch(JtwX zN<=jE&@cNh_&~Xr8&;Z@@5dj?lRmxrhT98b+pkMDK8y|9B%ebtsN+n{8sbzNYbqL3 zH!o{BQQphZAWqDjTZB^ku{0s$CaPoQ?HZvt=zA;1RK0SOUr$P2u`#`W{c5`G5+_|W zUH`J+CtTD))JVNB=EEjsYntK5-;ZI}>|K6Or$5NfSJ!1p8081*ncJ(1VAU1mPa2zg z`#qxXooC9abXU@O+4h9i_l)mbL{@2a1g?R{;PyT-IG__w4DdM~f>wnq6TG#7xaHy^ z{p=jMP2nqS$8kNy0WnV3(`RnHyAf}%d}=8G9aAU@4-4Y^(8eOtXP995kgHwO_Ilw zs*nq>00f1l{l9qKXkp<0!SujyHTPK zl%V>-_$l|rtknEQomj@hhlI9)^=k;ayB#W-moPj&a)uaqlkCfy={VI)+QRFfYhY1v z`FX86^jw$kd%#LH7ms*EKDxcy`vq~CSeLb#-bnGRO@4cruvkymTYME0l1z3LHM(T` zYH7p@m;3gZ(hTYNA$zC3K2@YHi|cF`k?T&h{1=3cz%d347b{ZA)!7}Y)+~Okv)av` zfLg&FlYj>#dVrO7?eBfl207jERxd{$RUupaV@^6m0TXU{^V9sr!Bfp#@6Q8QuMEXc zPK%OVuqL+WzkM+&n)YZ3ml-%NJ?T#5XIR;)Q=x$4>KV~#5j!7kh~5yqmBJR z3~dHd!nH!r_Ts_4Sf#@&4-|0bSddUlN6ur_gDZ9D%uF>sffht5nz8QI*X!LY(xTnX z=iw9*=t+CYJCzW+3Fr$5;DZ(%SCC`N&Bq7t7e3jV4DI!N5QLR}P<4DOR^DX}BLdO1 zKEu^){OoJJv#-4cCy3lWz~^A!7j%r`aD}?~xB>^02X%QSlWWYlfUhz|twP$&fAour zeyUt(TBN_EK&tqJHWehxWbLvpG5h(~`uEAb%rynNFV=NM7q!2AqZj)o7jHf%!id0w zDn?^Z;SOKq&W*+M@Ot;|i=SGeoyv%;CbuLlB^V!JES+|=62@xJKJzKpZ050K@Hlgo z3J-4~FDf|G`U2+@P=%kIM^^^qzk_RF+HWOJ^=1XYc(fuZW?UW zI8jo)?LWFmbzqi=3I)e4L-!CF|+Pe`8Y&GMY)= z*>2I9kTb=>V=Fcp+I$Vust&goRr!KtM%fUdda?!{?{P zJB*z$#y6j#jXW&p#9v=pk0$QsPxPg-4hmpU^|x{YQL5e9((x0lbw`PljsAA7noS?K z*xMl2Su*IG#zyQ?Ax*>lrif9MCQpcNW#pDb5=XMNQWK}OEoqR1AV){7`!PDCa$2hY zMos;<^cn6>Vz-VT;(@tU4khHf~#D(BUU7BG0h! zL9W0l54%-#Ep$Sf(w+yR%suxiWS?nmwro3CUM+*UV$)<@qrT$XV4@K?;nJ?`wyk7& z5q9+&o%S=%L(K+n}Gkka)2~NFQ)sKWsuzkQw06F`u#9 z5fXK*+4bsDNn?VUYIkj)a(g`w{`x|hm4{X4N01NyCd-@)J*qvx;h2m4c&V45EKfgq zXQ045%V&ayMqS0iF9%Q7?wgY3V&gPF{YYEx^+UeG&%ptYgCK{J$B(aPQl#Cbb;=oU zzA4|8dG|Tz4k@)Mitj8dP*4ZksXmA-7qYAy_TYCV|K9yZBJSBvvxen^eaHpbL>*Q| zN+4}Fz#xw=;$48y0K^~65YfJL`!DEW8;sgtC1=$&!o}Ii`0Aapsc|wePiS|^g%tmu z*T#j`k>hQ=`dBu6pz#pTkHO$Ln!c^yGNIJ!`r~N&uKy3!*RNXqG8A4Hh!$UpyC%)_ zlXW4=XhSMbXt`z`Wn)u4Y@RYDO{Uhwd=}d1QqpZ=Xi0bMRTRThwlgCc>NS>ChWOuX z5&XJr&NC}aqMD72u21u#2PsMufj-P$XT0H<&`UKmAR?{zfHFC3yhWd}wk03V0< z06gN946tZ%M*Xxn-zk3@M7YZR)TKC=vBTJFyZ%Mage8JoG^WZMIP-pP6ogkIVt}&Y zKkup_^!^25-EQRT{`%O_H=wfW%d^imiT&bjxTI*OXO^t6mo(=_-;Vz_kolL9%>TX; z|M&~yAaZmikPqNME@w9)8zHEn^*TEd;b)a~Z)7%e)nXrK>;70%qlowB4v9AU2zv#i zjywGU+)D0)1lAnlO}n83NHQ`rAdUdZ@bA*maf3d*TyrA~Yms$G) zcv116n%tgjP2KyF{eVSdYV%jcXUxZi{c?nbJ5M{`^wHqf5!Xh~@RTh_ANP~FsCwt0 z z{D(B_pR*$+57JRs+j#=2U}}wpz+RQOHUBfQw&yCT)zGFP#CFUu>~$lXw_FQ*k^_{h zzj$NYm@4pvS3+=lrn6P_i7uJdKXE)VVVo&J(<>UeK@Cou&rr0aX>nk|=!fZ3TD+@= zn{ZnrOC|li_Q{@6A4!-?6DYeAS|W@lel zeVN(a@=KC%2sy`5dI_ zb8RXIKP|wg`F`uHbTlGSUT3i$cT4ekl3#9WpGC(P^xesZ6S^ZIC!Qelph!!3Z}ZjU zgVVhM%0jmhnJ+Zxgf9;UpI7u4R1G>ksjRlR`yx~IG}IK#+`L|5v|todF-zoa8DM4b zJoM;FsLn5mFi;Vc`&i8*sf2qQyqvOEpF+fE>g}0l=V8*8ZgKt#^?>uk45e*-xtWTs zYh@wu)!iYgwYDoBGl|SR?{@#f(f(j2!`DBQPnAE>z#=AEVj3w?e+Se zEhvzRZ68Y|<*lT-OR=kdWcGBW;uXy_fS!6CuGtZHSPc?n*L#-C)_lprHkTJCI~uL~ zY3~Oq!?8S%2{-Xg-jUl{rTwTad1bx| z-+Omv9i+Kge_B6zLl#Wq8hg~~3ps62xEMfw!iem?Hv-=P#U`M)M}f>F>;MovS-!gj zJcy#PMy02hh6$V&;|?$UC97h)Uq>G5%9j0{dir>aH<|NQFA;yP4`lzI$xa-jzdx2g z%%kqp&7I>VzYm_{JSI)nO4FmBzOj4f#JCSuI-+vY=~~CZ5=UW36^|QjO7wK9tAdUL zc)wsIJ_;w@3a1Ut$1!2D4`sx_Q%+=pwLNk-!(||{O85!vRzOjBI0RBFD!`mTqz}+w zshPfQvSs6iAt-t;au_MJO00n+8) zG_iLSxaB)Q*((#b{(z9xeIByrA5*305hnq2W^JqpL5u7)dgBUgwVy;zvXq%8=*)=WD z$bM#R^RQPun_eOA{znhsZ&{4Ufsy|OSxoVkCNzy8kFN+`n4NQJjq(f6jcXa1#5`GV zv^TTnj1E5UtU+lQP-H!VWIiG4*S#<*jT0eg_C=yMm;giZ*`)T~b>f|Gpn&T^?3l|- z@6~_>WAC``hz6`X&CRXc?;l!58m&vLof3aUCBA&|b&hBQa-mwD<3C|B>S9cL%caG) zx9XV47L7?uyeeuaW&lPN2kHlAw?aT#iWlPmuh|gCLaVb=t)O{ zx(hAP{wkpNtc=Ez|A1X6i403&H-8PI7sk}9zip_XC>kUVrligmwdnlF5c@rowr}DX z96yVlJYX@+o&-98`DdUN`j`a;-;RnS$RM=sLRtyj*nkS(`<`Bb(3&bbRr>2<5|1a# zU6ddgu6*KMfyVf=aP*=pj^)f-gb>ZNHH-Zm$i54x>IxX14lGg@b7aWrl}vBh4pTVQ zYW$e8;87xpQ2Qk7pLdCY?-zok^FcH{7d!vYl?#;Ze@<5oLTv5HR_uuEa#IaXeqJOR z+Ixaz2W0oTK8OCI#P>|Hb9&sU9aa^0n(PE8G}D3Utc9C^@Qg)}6gToYq?i<$)C@{q zy28BC$C8$)Mk&hL)qF8dW_ut=k&bC^G!DMl>kn=|0=)6zqn;Cfo$t=L9cV3)GZYA+ z^NA?fvIPj%*mT-UL{B?Gt@Wy7Qy<%5h22-|sU2UmXLTJck zI&ObWXE;mY8I)JG3Mj-!TZ~V3@!4Ie6uXts#Zk>kg#rGRgN`5FMpELbT#tZArU_oj zf0~M5Ku%T@65A~v0zV^tWVZ)S`m7GQDrE3S0%F(~Qn`96wh(j6q+-eO#>f)|CHc6v zXQco~9TK5-Gz`4jUsUNNrOcNzb>|WHGRkU8)-c&0ej?-l41{s!RdyMnMG*9U0y_ezrjNyj*EXG z7yFhd6)T6ih-$-EUNY5@b%>{(DMXgB6`R(R9 zl;%hL*{N?!COdqi+_m9mp4!rPVeC{M`_|-IhI}>gPy3}b3#vp>$VZwG2t*;DxUeQz z@Ez})1gDf;~w4zN0JYV3&H%0;ft# zlzx#2r?+KKhXv!1JGEYpmevr(FNUXY-%u4PMrRVhj!Zz@A+!MzxxQq^2??4I8(EE< zPjm*KiRU#NC`dra!z8aEi4`d(PsQqoB+2)shAIW~PeT)|b`o!w6{$^qO#C6!rzAGZ z4WsjJC8%w>5zu+(o>Yd4Ffj1^XdZP!K8aizM&+K`oCvoU;?quaf1lnJa=Mv)%LnW$ zW474Gq%-2THub$>2Z*RUxCa?u*BY;$_%pvOS|>5=pzqCQ>T#C8a4XtPiIZ&eJg3*W z2R@h(;#kr5h|stLM(*fOE?8teEolAv>6U&~xaqBW3kS{1LQPIx%~YKq_u%)zjB0V4 z&NCXrSv&-Z{{!(e+5mTlSEprmzeVSabAQz!YOasQ7_ZfG>=~N-o_*j4T$o+0N17M- z?5}kFeW}OnGy1k^*H>N~I$(VZV5oK@f^Cug>EOc!N4L7H-I!Vc)UU2wy z0?Vq56DwpF**yBp57dM!%)`6Ffx-UI@S0XQmeyLE_{IR)Z7p7tDW+XUo{LmPHEdm~ z&0XAaF9kb5Kuxj*Io%H+LOvgW!I?>OcmTwTiP|98JvBZTjcs9aT+!t9l(yL2=6<`B zhYdY0x27IU`6PZL3g-&@0f7#W9u+AurYGEBd zVBYVQz8P2v6y=2Eiq1x@o%57cr?< zn=&_v_~l5m+vds}{WVe+Lbj&4rdeXznpS)eQDW?1qCHfVIx-LzY1d1Uo)ON&9nwq^ z(u^{}_h6$s@)j)6UQe^89i+StU(U2Dq_MGB$NpqI76pT5_qfyDTG1Cz1AsN2q04;F zI&X>+`_k4ePCc5O_YDx(!+z_2vK0f66+m51RM)B}_?uDm2zzP$2@3%g_Fr_Z*L^T* z%^Ua1sbbZPz4Z~|ooQ8Kl z<-=ju=YaEo)zPVl>|UgZyrIUDL)3joa%Tz7)sDjh1JL zfn4~cR@sBnV($>0qzA!TE~TAJO@ch79xt1bwHGjod9iOAKV@EMlsc=F)LOLa=JIUq4f+mdBcat z!w1HPtX0|HCsQaR+KB;e_|(H99vPufj>h4eyfyVHlgnu;DgL*v^B&#?f6J>tdhfG$ zMDxOC)|d}31-Hd?<;2Lse)M@}^=%)-%!zyO6SKsjGd^{E@LEluey z;uR%H1yvuq^a1a;b6wGyXKEk=KQtUF*$+@#_zR+6lel&OFpytYdgP!>b)es|$DnaA zPz3N*90lC5JFYG-^1V~U$pu=!&lv+^pJHo-7-4cEXTK_W-dt*C zNn^tWx}+8UXx>KbNKV=p*t1XL^tnZ5<(@gy|#ESP;poyrzYNhV0HWD!O;;_6t4pZS1fC+Ea5_Zdm|nG>~YAv z0wN?N$mihOVt=%`TuZMS*Bb|qIdix+|1s>YmE{M$Z>2kMsy2Y$9q zQFQ-;6j2^bivF1H@Fr@3%U0$4?w*qgp~b0AdeThZ)+JY& zQGYHvKK66!SnY1up338lvMHQ47T(&;Q7EA{&kH#l*pE`S&*MRn`(u%tP9#E8~S)$AHSUbSU_<>@k7|M#UbJ{VI)vC zu5CstN_}vN-v0Q`TfPSdxp4oz%^=x?c8IqDpD-Up7+`#VYyu1ZM=e3tiKuEU^q-Ob zU6JG=a`4>urMd9;+d7gt$yy0ho4PWqy|N!q=Mlz%BS;!~?F9sry~%Re-l_-3Q|l@n zuJ6LU&M2Nm)S-sI?T}mStvLWO6v!h%pKc1M;)B2h5odqWVUfy=*MT53I9u1&Cz8@i zgb7Vr+psD0LY51*&G%i=Wc6}>^YAyd%n@IY5YCmaBVIez7`z`ZO~+5i-H#P4`v=6= z$az*$X?lW+Jd_P+A41vKkRWV>G7=SVg<`}*>3Jv}WrXW!7ZQxCjUXkWwV zmo>Z*&Hb%a@jRR{jH}k#6aER}Kk6c1HA>g#BG@i}pNlspvP@KlgszMG5LJYQw|Xzo*n zrpSTR<8fqR^%7V4suFY>tP-;2!%eZT2nQHz<%)=Nb{dBaGl#p?PTL2@d39$^$)#GS zvys$%(^>mbdQ0RZ{vP@M<1{NXZtT(aUdRZcWEMRe}zlJYDxq#H%+;Y5E9EtkLx-OgT2a z|NZp3DVNgCPOe)aV^)9eXSn=m>^aMCCa6)9Ilq6;f3FU^rnXC>Bu={k z>Ff(sUj67l4v?5RwbTibNcXxkfi+Xp?Mi!66d`dY*jgk$56w1ii zICm{RnUQrOsT`o9c=Mu_W7#jW8aycyZZRjkMgp`2l&`jxV4tGfN_jsn^MZlUf9D=j zs0FSYB{ohnP6P8#_yx+CDIWXM-@d&L8qT}%T9_@jd10&bR3(7=g!fbd2MgWaFE(wH zqRne}rI=Ku!aOfhAr-DZY1|{)>5TF)opx-iP09Ld*6raVv1MUQqqz(LGq@q{5X)); zg+uCuPP+@G;wQIr#|Wu|^5^ovG2&~j=(%IWrA1;Bf&l&&<`=xpB(x$B+R%9SedLsh znsuZqKX<_~^fdNOwc=%eZd|#)CiZNY5ayLTK4#JCCvlkYZqQm^Fq!jZCoA#nH6fF%e-LysF{X_BmB!_~I{=8VX(TREv0=vor{ETD+ z1Uw2jwq*HW?hRrSP1*Mcopj)n^mD>m@e7$0i4AJcHo!MIk4K_n03KN z4YH^<;L~#X>{fKuX5(0&?@>SDH_+q6A{T6*-CDH)LyERA^>ByCH3_=-y5xIIoKOw!Gv9+_ zibk2Uh%0aL7{o`lC8oh(i4PC@J05W}4Du3pmZD1YA*fGqdt@|_8@%#Y&07g%yKLHZ z(6Z(Do`;^9syFLG!mF6q%Z=*y-S*7=jZW-U1WvUFuh~uAIo7}0^=R`M{SOVY8je)B z-ntSi5<36`I2I46mV)%aG7q3$*p5n_KsGtSJF28ab93@vhw?o8p~Ay_5v$JqoR@g* z)<-vNaeyl3L@@a*PX1=@#FO9Jj3pGDpHq^k0?2;K96~FInt-lA!(SaJ1ClccMCt@4 z&{y4!Q!&Cv<*e&^hhA~q6kCAXN8BMdG{;|5hm^E-ttYJZwe0V^Y)DcSvP3U~54-s+ z>dM2)_x=k?3gkqoJX5})&1w%kZ2vkiZPFD7e|;Jt5_T%?RD7i@f~u^W{>R|0&)?@k z<#ro0J4Op^U~?3&w7W4T&W!zHzx^ztsGkh7ll>|o{i7Se+Z-?PMPZ(m?5M3IwmRd= zZxD@k>m{s-Upfx4fGffB6`6ti zUGU!v`g~zCDI^aaNWYji{)wHs(~L==VSDV(Jp@jVDHu_pj6FfLJWL%Y{8P$;Ct83< zqnZA&>d_u;dg0JAk342taHsTf;e|XBFVs_@X(GdjP^fSf4|NxfWwx~YYI;C}AE47g zKbL^uascr`KK9qZ*LB$kUo7XZ*P~=e0US=qy*g7{=x1z>;#d!{)04zf>Y?mjhVVa;E%VCXMt# zG=3QxoNB?0eLcA!Oy!kYcrQ9YEI#7)Hx$jL_=k1Zy?)I@2VeXh8bd5XZ1 zu>JI62q?K#ZPykONYq&=O?DVIA75CScwDN_j=#_{OyN1Bl1=w&n@ z($Du3dWHXA`^k$SR$i+!Hrkr+v`m%QDHy_bQMcy`RRGaNpF?Qn&L2oX_lcyx!7u@s zB2PiD#IiID%GX?MyT|a$Xf{-Ul!!z2u&-K?t%xu%BUdwg^Vy(nd1;x~H4c|@-gFZ% zFs)lLw+=p*NbZRrKnm#aeW_ASvY&YDlN}WBLLIu$h)BP|ovvXUgGrIPzB2d5knDVT z9po4>Hwuny7DvQoi3XlFVW$9_d<|i2e33Q-G?pUPb;WT#?Q{)KPF3#z6r2A>I=*;K z?xOySFNr$e_EEo00K5jb%LtbZYwf$J?DIv112JP3_4rS`^W#AM_9_q%TN2`Y_hpaP z4~8R+@IGI`?2QKGpCO7uB zQm(yJYRaTx-#$phhYABzd><>42Y(WoD%9wLy>&IZJMX7E{zltw#|HA_jM@ZUXuac2 z&07N#ee^r&+xo@T25kSpM6YNxZ-a)s)6dfyD3i0~rzwb7cFJ~!+B>NtJOYVy{t+zv z{WfoHhhEJDDez#BooFUJa)J3v7XHP=_=Nr9-42Zu!{o>})4MdX7^b#WWJPtH(>{lf z%a%~dCS7w1ZP{m^s3~V+o^`X?O*X1;r*=5_2aep6Q1_{9XZ@&P1@#7>Hof{XURNIZ z*gIeIyDb4%L%e^p1+(dCczDG#;)}7=^6jo(0pBlKH_VxvA75^n5mjd8-emF(l9>$; zyy`;k&2K!aPxSIYYRMqtgPm2s^|AG6N8E|Vsp`#k4@y3LP0EAYy<~a!jyo}J;#2-G z+@lw2J6+GaFGW87cv%M+S<8a*OlQ0r1Ez$yEqs`!Z0m7(G5gFh-}!2^+CzKYNBOKW zQ#5{-jj`?i+9zzSf~N*`@2xx|B)V0;KdjXB#*%M-gmsNhK7u6AAuIaY8{C?t(+rd6 zE8cK?$dCD%s-KxeL2mv^qf_tWnc%D=<}b*k1DkqOy@E{4X_93Z?}TRxJwxr5|x^P?c?ce)rmT}o0q)ktcduq>?3#Pxx6%qw*?+52)%Awcev$k z%!$Ozvu!!&)(nx&oI}h>UEuS*P6?vr&Vx97OyBAKjlQ!Qa58OWl*8ITxpV@r&-V+G z!KNQHM&@r?DnlQAs=N0p&VRIRf3JE&g(?G^gimx4K25M?X8~El>_^*CKxM6;H^Nf< z_f=5nAeE-}-Xq`PmJqXNX+``;aV}qpkiK3p!7e~FU)8uqL8eJat^V$H! zC10e-F*XCV&S%orfFd5U3y+1$V7*>fL}?+}?F)*82e%@dCbQc%*YL=B;43WJinEYf0l4F7bq!4-|P^w{luZ^;{Pyv|=NA)r{(ezd`Tl)!zVU zup>aeD~}OMoSAr+xgcdIEp~+`UhM-wjTy{^r;Aup5+|lxIS&T1Vl}dXNi~=>idE+N z>TsPn0i;WgbI43| zjy;x>*3?T`wD%l;9SnH>8+}~f%dvvLfHV^LCM=)51TptFxrMqCtRRKO^YFm%Fy$fd zK{Z{`?|WeaE3E>pW5LuqW_c}fM|?BXfHIKGAcWsr+f}+eYd$$iDP86Y?KMPcW@jcqL1SOQoQr zX%w}^j~iw@W)BjMT$k&dUb^Cq8r0UW3T9CiiC@|h1l@Fhi;^aQ@dM#bknn$6qW@0C z{#)Vw|1S;uSB>%aE0{xn@x#e_!yTj|Va@M-)mNV+4fCq}CN_m2h=IdD(vknLQU7A8 zvD-|61L>FhB~KcJog@{Mi!*wBHVx#Ta%KJk$wZLh>16h@EYJ$$hj&Q;acsQn9}8`iDnN+;mV(4 zznlbE7b8`vO0u5>(~{6l)FAPOjidm!lxv;E-wXrGwxo_x_aZfq=AeY&702wyCiVx8 zDVHbjmU8nadgfnfe1X&HaCuF;z_IHwm+$euEeALToXAd}?%bYY>#~%yJ~kROg5x$t zPQs3i{=CYht!$+8H^5sJdei74%hC1WK@*) zcA4KQXHG^^Z|3WL>S^1P`&wZp4|CrtT~TqC|htv45u|We+st@FHJqwXif5`C6RV$uXknM4 zdz>zoe8?H^7+{PPW7Olm_->AhXlNwV4GDBciy!fYgk~eeF<(CMa;JR*ap5+GJ)tBUmfSFr~<@=uc$yBT2)l!;1W zNr_+h$x3jd2st`-8XDwn3D&kbFAI-tLiFim&`)UCI3;M&?STn)W|4u;IC=2&9*4Q zZg%p1lI2WVKXfem93;L6qqQTzA|LMw3$$v-8d9y>J13AS=Gn0o-vcYYe7}L=-e|Gm zq}kx}N)DSB>r7LW0&iA7it@;(^?weMf#VHl$o^hT8F=Bp7t^QWEJrJ6%ebz|@}3E) z%?`0R9fy=i$YNZp3vqfQ4R)JA`Se_h?{)XLdjrLAafP{5&x0!A`gZ#ZZJ`Y>GO0#F zLBov2)wA4F*q3!LUw_Q_H$gCFm0Yqn>ag?_5_cMP8p81>SAVu}nC0kYf zTR-woj~D%sy*h0djru{!>-Fn`-_zKVc%Ah{)l|((0d5^%oTYLuiwyfpiuWcYN2l&lV%X?$mZYG8jp&$;>qf;Bi>>MDTsU> zCZjcj3P1axtOj>2KTV=4u1sz@LD$JS-zy&YI<$Lto?b_UA@&{?ff3+mXpsNwoK-v9 zvUk35@?TZ_`6NY!8S|}bPTDHhb*9l0_!Jb1gh4rU-oUV-+1fWlC2l{X=jv?>+g0)1 zA_{Y9X+(lpPK{kAM0#WpNKBD952W_ccIv_P#_ufES8bj;^X@*Fr>nnvo%YZ(@%w79 zs(;Z>ZAbtPD>9!Nddfl#+LFU0Uyt6rY1-vCl)$A2hRVgfj$S4?T*YIdvUmpU0g`dn zOb2sY;)X3!vWPeRvXyW*T#mGPy|4HsqLgoF9TVW&ugzT*Aot6znk%#|pO|itTa4`x z4EPbRN#AX+{dwrtE%D3ehCQ$4&smu@9TUhgYCf--7P2}cRyrh9k03?6J)5qPi1oj_r##p%~O^s=1PhKoqfh`;eSSXG*{px4}!<)z82cZhmbP4xVH+JS_-VyKq& zqhp>qmNCvL;j6IqLW0v*(PKB!>p3rZg3Pq9ogjEHt^IZi%gF$|sp0 zZX9~JI^m*P;m^`E>L;@}HZ1FgLXvN<(E>6JjziD$hxqHECf7}t!?UoPdn;C6DRx8V z&nbwA#$J8_^3`Yce?h=V0qmB!+K3nqI>xZOs z5N+wbr}k8BtY*%(QNaN@_Ib(2pSrkJXh2?c_7(jt`pb+YL2}l+?Ey?mPzCIlIjJ!0 zVVg*?I?|&6A@oen^ZlJmUKdz{fb+M^%B^*SKwd(hENzUxj%x|E+O~F04Wu{WYHin< zv|B%hY13VotT#<(qi}v*TW!K``sMM#j5O83CVgf+?L05%D}<7T`Pj-a@>77Di;c}C z3Y&CgcJ3ANpROwc)BZIlu3$89BTbcYwyC|vmU_6@rH#**vtwV!{0fbRtCMy6x?h8i;fl*Ch=34*ot7cLHw>m+=#Pog+mc6KO-$RsXFp(Q9Ve|jdt9xRCtlv*-b+vp{YK_FZ~Md(Y@eYOjquKH4x_3ujVNDO#5re|dwEhhEI;#N+SFH(Cc8@BABj`WFQC|Mf~PJfFDWJ@<-T zGy6$+q_}ZJ6U283{YY>gKJM1U0u{iW<(A4@+sg_SzTA3A{4-B1Qvs`FwIS*|I{o(C z1}0;Z-^0}wtJStMp(8~9Za%vH`J+uRQiqKzGV#(DIU~_9j1#ZH31hSa4D+|w@KXba zRoSXfMW&S<2}pb7g_@Y`%J(+9O~e(+ytX8qevzl&|DN~#ezf!W&(~idhKl~czL8@a zdE8RUMNagNnD84s?E5&Ejn4ULuQuu0PbI~Ed3<1WNp-BGhoO8a$!@Su9$xyu7JC?0 zuW3_gwpLvmU*RgCd{xN7jI)66fw%lZ!`43pn$`oIB;wlg5uADjUS8Y+`5;jC8Q-8uIoI{>pWkJ6X8w{XjASB=#Pl`I7@jI%uHEXG~d&8yV~0PD&+t>7=NyVviypEL}cxmw(HFZaCetlF$eW&m&iZ{~7?Bsf^>f##~ zBqNAsWmD@)D=%ztCwo5*pY7;!#wOK47J8P|$rJBdO#$73v0Af>L3ZeH#}9}{wP7}z zHYQ|oOpwhKoyGcm`knT9;r4pT%|*To;#;-1io-89K509>qs~PusA002p5!wyHerjO}=ek$nvDMqSo$3$IXpb-MPmOT!X&3FbeG z1YqVPN~pqS5)BWoQBr(YN;TXcJoKBgh$b1aj=di=H(eyNWfyi*3zi z$1@+zzQl?&Y9L**1ss0GN0#BfnLig!w`{Sm*>1;DKJ^t_s=Sz@(<{{Ec5v@&UBqHY zB=f~SZZj$`?w$m-E#`iY$=wv4mNV&%#RvKIx#RP>%?HCeG!^03e^eNm6lgqA*ih$Bm>;Bk0%T4w z&w*~w-oF!_X>;0VbEmPc72T%lrrK%a?=_iEEZd8|<=7`%XHcY$r0%0W0ue<~fDK%L z;(1Vck`v8cDgF-zzE9yB-0sZh2LC1x>_U=}c;PQmOBr1?M_fIm$K@$l?)EzJ#5WO- zRs>Yz9f{%u!Fh4XF4KTF(Xo&4bTcoH@5m)%F8o|mzkmL;L^>FdUbPljuQi@^Hv7Yt zs^j4W-5SyrWwp2IhNkw)4ib49a>uc?&w0lj`F%%AZJx%*q$FM~z9LXz1l$~f$*uNj z!|FemVo{pztc!%~)~vCY%_uv-U^}I@hRx}Gjx68|!bfqR% zKY^X|eE#nH%wLjmIqw(?hBWS{KQAmMHtw#||Snq*O0@*ygny+n+5wY?6C^71C!WOpIfp!bf=~Js*@@^4?jl* zU4e-7@Y8!)t0LRB2%tH4(`4o37<_-3&rl>5!n789tbmzUbaP!CExarLT3zZxu4g}E z@|0**`O*9yDjT_w2NOmPf{S440h;L998*pxE#;edx7f9#;c_%WvvkI6$$oc??^>dD zwy%;M&~1lvH4E?^W&yzdG?EBPa`Iz|AR)YP*nY5?1^$kdd9MneZOq{9lzOhT{>Y>> z^5qr|j8%^PHx;*I6~c*%$)}p+0*%i<`=oVQM*Ivr*rkNeTDDTyQ`WwfT?PG|LU0;@ zN#p&|J!__!lSU1FE)dL)Lo$cLoxC zG$aMYDlz!o1E;U;Yo>k~cP(@%euvs<(bTa(mh5GJqpP(#;TI7GodM4x7aH0NI}bD( zQyfO`FNtk6ggjCsxBulpzB;*^m`I{RUwi)}XDYhoSNkYvF1T(C1B3lq_aKG>8V#3-E-A)P8&)uewHxpdU zuxNt)+{?K=u|y&RSfItUJ@`f+uE7!~7Y&qD?p&^J9Br%{8|AQbFoj<`_tfZK>Mw7~ zP)bDw-tD`V-yT(GnQ+AjT;r|d$Y*$7oj5U!70)g7a&DKd|k_pevDy{Wv|WZN=JU769oBgU374!1v6=Wqx$nlMCQ z*nc&ffP3izo{tojjXvII18rP2K67AO$58DCL0n(s3F_uKIq9C{oH*~sf0mlL9D1<} zF&~h%{|%3si0IDhP6%T27k*)Pe_Ko6nhbA0`_Pc!T_45MY@069ClHaG zV`s<9Q2u@%HL=*egDBQh>dNHJc+A}=R%>K=Q$(#$fjYrFHI9CIMV@*=jB1*4xY#*`0ciuXmGK7G9D!^O*`$3V*H^u?w z2TVJ_HdiJ^CK8J*PP4`F5kQ?$k~ns;ekU}1hlZzn{}r+w7NieKBHO=^tE{KNo&HZ{(fIhD;W>jyqVh+?1FdM?!IH(QFE4_MnhffNQ-Z%c zp8^wlh!1cdyW>eb>oDFgbup8_c4T%wykZJV(QxPD4!yl7pa2v;V@?H7PZ!|dZ{(k2 z(A0hKkf4LLrf?1@tH-{!sqioT{z@+VUfNGqt?R6sS1G?iIH0>ZkDeXLfl2`tse;50 zcYi^kP4wXRgXM;}qN9`@1a5r35VQXTni`X_fa;dyLAH6lYyIpjSu|m3spTr(h|f3T z314D?giB`ul2puL>cJ<%M=X*Gk*zz}DDQug$2BRwQTeV|dpY+ORS&<&M4o4GaXWee zaf(}c`+1vm0-6gQgOn3|yhOI(MNG7OfMzn(IVeT2Uq|?iQRbmE5xNqNz*t)xvXMCc z{A-!urwi=7$@iQD{R*6nniwoc1Y(pM>U3{c*hwruHPQ^Qf9A1m=?Xy@Y}DhrEpca~ zs{ODI*y@H63+Mw4pXB%Ee~A6xZttS=g~PVPwyLTO&zSYWtD;*IS%KhWJeGlGvh$O= zA2l@T<@?BaJHy89N9-3s)V(m6_i24;d(s%Y5)1h0XZIu-oG9;9S4Of|X_pLD7P-{S z2Y5paDYGAd!iniuU5ug%oclbmM@}Klxjf2iyixB$-*_gN3{|gk=>s1F!9e3j)?;N* z;ZAbg*xXgAqG{kz7?6$|Zyh`3ItbkKU@q(!zfSRJidW%V4ZXYOY*ce1;Rhc6;0KHv z;b#5gxo=iN15dpp^Rr`kCc|rA`yBw>YBY6wxrg4w&^b+NXZOe-q*@#{__@qR`q_f5 z;*o592Bg+)Li~S~K&THea-!k5>~Y+gRE z64Oy6DHJ7sPWknsXR(am*n%Bo{(6-APgPAdSiadu(!d-Au((IRsF}TlS1k#6Qj?Ry zz9_9FJuQ>mh~Q$6gh)a29+CBmF0;+?z%Dc zA(kXDMiR7efboy$7upwf^Y{0B9<1_c40~0?| zj`ci7nPAqH_8ZEvGvdqVzQ3`R7HhV;-N2ZSN>@qHr$7*TMQ~@hr`U%Kr3r2@)Lj#)Mj+p392X@sIj=K)1u1#NUiU;*vsKTsQ>c zQsY9Y4!>DX>|$+tzBO~vleIi)u4(ogK{|eOcEj_woWD_en2>F zZ}2%hpV4)Xp{ZdnvQme5WI>MP>hj`9OkaVWda$+n@UH}}-ceow{=@spqiv~E%D_+o z6@y;zBc2}P0&t8wPsOz+ttp6R{)+}tS+M%gQ0G9Uue!@|qRh#uzQ7#~@Iwjf{|>eA z!+)LOSuk5iM_OINi4RnKCcG2bSos?7`mTxzT3W@^*(bn4Ud|z0=Ft@r0U|Jv;@H^4 zpBe8k!>?LA4EUIsWx-Xq_4tK$Ve*c0l#w(racmXHnjq-n7`o;17^_sZtcMqes!o*F zL{~DmF2AmiAARmrw6y%9OCQ3al0*I-QM`yr+*Z*j(tT3eoagfNF;B^bx7s7_5U~%r zZ-Skj7lgaExpnbe>J?+Mz9S_*G(9CBqY#4-sE_4*5yiCu|H2a^v!+t_u(ZUz}%1X5B%p&+m>65K`={WyrbKb zt9{q3(>N-sF0Q#dZ~XcpQvj{Wv;ltU&+-;b#l=cpB50Hy#M^@xm>)~;cm8AEN4|ua zde*jD4X3j%ZH)TPpThcD@>`iXfR;3HBR z;PM$#(KfMMmDm$ML)M+(NSB}p*j#G=^-DB)56?Ka5xs1E3K8w*HR#%u*#(IY;@q*} zWH!L#u7I_s<{g-7$XP+f1Km7BhR3ZAv)P<~qwf zu|SUBj`4b+H#kIvZGkjRT3G8Sn~K_W|BH9SpPG*g-i4Yzg|(b<9)r8;a(w{cNbBmx zci;4+_;n75kz+c#62WRu;PHM)dbC3xgEOy0`Oub%0PD+xaaR0H`_*qZE6DDJapqhM z%gOObj!PaVdK22;5#fy$LuGsFWlk6K>*RK>B`Cb_6LIZ)08LVZ_$ms-g$D7HK_OJa zp=U#4i08_`4#OA;Plrgex`D511hGkvGbH-2!=7{EK=t7GhEZV#H``CUqcLrlNQ|)B z#lmB<=;Lv1d|df;b;DbCLwe8ONED0DxrCwVQB(U`|on)}qwCm4e)P_xBEf#cgz>@6F|+J? z^ntC*ttw1nt#d`1Q=CckN;}f4W1+B!ca6r2l--;13)?S^Z`KY8j-rgD1jo0Dk^z4=sr(6y@MN_0( zzj9SvvagZ00;hpa_8szv{3%+o3eLd*A6CLR0&#Ys z3>sw&i`8jk?{TReV_88=4A_vpw2r|H)$7DN?HpuNU zp0Z7Dax0U&yf~G4P-l$8j4MNx-Se83q1MeL2wb*@KS`a!=UnsilUNstIjqp9BEiO} zE^*lf{~BW0r1rP6U~N-yaR~9&6UljVy&Jy21tOQL%jPEI$B&KIE-8vKouxR!t6ltI ztwPv6$R2K##}=RP>EVc9KlmE43)VZM`8n<-tuc3XglbB6xd%SVnq@xuR5fGWA!iZt zX`#1j>Om%j=BfnoFGMW&oN=F#rbiC>%;TslTenNAz1 zkad?r0_j+GrjgZEyiQa>JeRxco;?|fU{p?0d7HYKY}L1D5EH-`6~Z-z=ynwpkodno*R34S10(s zaj|i|ZXP;!U7=N2hHTZ_td9>jP6cjg50LYNAfFw)QN|X8`Qg48@bn(MuBx*Ya%gNC zDgm1LkG!Gv)bwggOlkuVpDaO;aVD-2>L3>OCJ*P|;qUA~%r za%y%D`?#B~?Mdc-Y(^mk`NY#ou^Yxi83LEX)4^KIG>i9{rzhzsTku2Uk(hJ^Jis~M)Hy-SAxD8Qg z3X@vgfe|~Sfk3z>0uzU9f2fss$TSB(xX4Fz3WXKCYn+tPq?odHQB9SJu2~Ns8LFGKkk{SXhze{ozL)C`}ErpKQHF^ zbl-*?IW@DLl!-}{lCD`<8n|74VELf!(W}ICy^2T|J{u{k30&Vn@^Ir5?35lgSoQ4N zf$hsv{k3ohI^ideF5Hd)FrRTAr@5}V_({HSm7Da@lF=EK%`s9?;Q0*Y3cX4uGWC)> z2SUl6ngoGeijhyhl>oI*?lCdp5HnCa8+`xxhQrra$Vyb$Z3A^oLY-W% z{&1XYob4LIl6JUsMeK5a*;8@fExZ474`Q~)Gn;4cyEg8dzMFK7w&Gm(uq?NdW;Sgx zR4Mp_=ap|?+VUgT=tamT#oXu|MI+~2ObzLPaMUA51Fon3;fIfD9sc9B>b%AhY zT)<)iA?^4k0(lD`PfDeH{(hV;BHQsfQsD;IXjemS(57Kj8llv z5b((Jmyq~ar{ay>JQ!JgM3>mWG z`HB~hY+8x9^kY8D;0o$d<`V}$clsYpDc98$6uS_rqN~}(<`Rn+X4IMS5p`lO*zL!G zKAxEK9@~Kfs69id3BI$S_VmZgbugiKP4s;Y67duHd5$2)P-! z?!vwTCsTx9t)t4Q1BNVOlZU{csNWO{V+2n=n~4?I#2D75Pp(~=vASaBd9-K6|}_TX~Hm$q#0R45K zxCu*m_kZdr{*NQ_5_~F0(cblbz3FB=H6@yRoe?lp^A*Ts5`kRNqt& zT^8WgnY4BNurHu`L=8DiU8p6#vZg`)GzGm+E%5-%CKy}_nt?|xSFt`)a@xa0&O!*X zZi~vIiS&rjb&D#+l-bF(WvF3$(!vc1;M(Ug&H+h4FQOtyJkAmNP56U+ne10WBG0gy zHYplQ8;YjW`8j*@so~c>$#AN{ii_Ka(uq{a@eb=F=8^XcD=m6!lnD|TRXUKhk)KiT zeXPj|_LjJ-2piq=@baLv2i_6n-&zi>I@RI{Q`~?k|fDnCFAB6FwrF5-+wqm!rTOCGUhx~dl|%L+}T6V4N^gD$%X5o zp&`%m)Dr+SQ`p;8N{MUq)2Cis{(yrC)xhEo`2<%m!gQ`#?G1T#Le@FgdS(VURL`PUx-E%SDc@`yUPY zQx0$Rc*2ZsJqitQSO3FIM!8Az7b3Kcs|9vv@Vz9nexDvxsTZ8Xg*ZhcHwj`y6E_w1wf(>A?k zK>hK(c-Wv_YBzL{5|j4@Mahsda@{**){y75)MbwA; zw2&3QjOu%5TIO^071$|OhD9K#G~_enVG45d_*98)?eii@kf=NeyS#esCn1VM`WeZN z?b84aFXB_fQSIHdp6*n zX0lGA!PTVB&(Tjge5I=kA0W%gwbE3z#Q4I{YS(}`XZ+q ze%u@}&cf^@7&NZl#vDg?SyzsS2N`3zN^RzyC9L_c>D^q& zn`m#Dq_21URCw@aFCFs>%2p}RVeT#V_ME$amq#~1d2pT7pqGHT#RdnsvRXC!syc6P zC(K^Noojep_>e$4@h*tD$LAwYd65J3A{eu6v7 zCX+GNP1wYk~e8T2P%MWIGC ziNm?25euYw2^ZNwe9mW3Ie7tJdWwgi_@92ttfPZEwb=^-~GN>FPx^j|v}d?tbI&c98$4cwWM%1jPjq z0-o?>U!4lYP;%RsLV~!RSJ82KJC0}u7rBDglW{8J<)8 zt^LA1qL$ZYF6g9YVVAkH;1d61Kj6|=Opn<>j{t0rSyXDo`cKS+Zs`(wEM+fDz z1wH!yCjEBv!Fs;dXH_oX3h&@$YG-N2;5F3X*VfO_UNWHEXWGS)<%Qo!1!ZJ5x=vVI-JE04#uyrC#4_} z-|m)wpDXalu}e{dNuUam&-f1M5C){CE%+X^H18}?k-#Gy<2^duc>AbxC5u*$%}g6V z&|)Vl67NTU>=h9QUh1(y2RbU3mk|5d)fLr#wby0pZGFw7R80mGT6Hc14hml&piF(_ zrwN9sf@8?_CFH7|3Fy8=pFlL1P0(uHM zXj7N9vrZ#8vsjYyoc&}nDgSIwZ%eO2uL3({+d;Nx9nBQPlkeoYder@*>R9(@MKcHV zFauUs=)ebL)UZ4{GAC>ua%>r8ROYtG%!+X^(N9CW8XR5?us#WU2lzX?-5z}_dgRD+OzV)wVtmGU(%zqxzI^#dyd=NNA+ezYD0>S;(&fo z<>{%++dalI5UFCp-O`Yf@))ocO7N%?4?roJ9NoRJY5f6C7s5pdT$s)Fa9kijg_dg# zJltECGk)koV0Tl);yeRWoS%=upC+DhvM3#0<1y{f|7B1)c_)_Hf9=9;rgMSzY$0PP9&x`nUU%qfOr29Rj5LRV{2`u#Zy>(kD0TYy}^)ru*#(I z65X7Cnb=;T*cI$wf=bAx0c??VR;B2KYPSu`+uGm6lI~Qfw#C^Zu1-u1a?{uKwzN>T z?eBpu3K)tC&EVQebrZ-{j!&S7jE0k09Zj%x`LWRsi7+RP@Znj^PMpGp$S}kr=@2Q!DPYh+$xNcHH1O#C@S6$Jy7Ra$yKr>|Joy>o4reF zG8C?)Z1(??BmEtSnLxIr&ghAGt^2MDREkMGe5>86IN6W{B=ewoM$Uxiw$W>j<^K$M@jNJ&!1r8yl~NYAXn*CXhjLLIZ+X`Ux4`xoo&5tOX(B;g|Ay@7aoP z1e@l%&zydp*5poRgZIq|d3srEepKAf!T?j9Pd^4m1@z>L+r(rH3n&yb0X#i2#Mf*M zR$4HzcH}yc_i0MY=ZR;O;^U+zmIh9EMfelUnFcAgterDDkcZu2oBjUz3>+u%@TJm+ z$e$8!C-h3Yk}-2T_f#GOf?X`;gclT!?ZCs2o#5l;R%r-<$Q5$Ot|-T@ur#Y*iu?tfB~-J^!WtYy4tb3-rk?=LR0ok@IA@ePk0#y#t>_ zk+h@g!T%OV?**8Znaam`m-1QQ2Hx6Y&EKZ#q+EWcG8}2by@j!nN(#IHgwZ-sAJ8_E zhg_!3zhK zR?<1PW+_Y)B!~)#itBHa8sMbo&hH2Ypx^~CwF8byaQIiPwFtWlbFF*?-D>G#MA}~L zL5BB%DZs|JEuB#_=5ob6w2zyIC~zXF8p&|JQ(eBbR_x>Y^~F=4vSw%37NGYYC(d4d z{_xBGKZS89!`uIc*!>4;cl!O1?O@&)1eu67{FZCplgyfxxNUB1^Al7(+L7UTKwgK( zN2}jvrmMkroC0+1ELxh1!tRtwT6azz9@?)Vj!6LyXae&bQ3K*Lc2OH{!=wH@fop&& zsarW@I)d+xhqc7`9IN&D`>->8IFh9<+eP8mWKn?NEacuJrGQePL%!mN3OkKYby7;puPuT{5j&YRz}BUWlv|$Okaw=&f)jq#AA6P z_h>WEno>bAmZtbApu=qZXm{xx5i*Ho<@ZV>XWvI%!zXU^Vta{tMVPm1-3GQmf!wTzayi&N%N%%`%7CW2LlVYdqsDe57*AF=7 zqRUD(`c2!0%q7g@9jvitqqQ;%xZQ+2BCnBkMd52)d3+XQsLG%G;MSV>V}XpgF!ALZM1m|J}@lQJ)mg zWG_A|FX=bAR7*|uTjiBXh)`pAGn#OW`c_FLOsv( zg;9<*{+c?l#P>~iWUwj87*>Yvh__3bV4!sG6?iJu`0fXZ`DQF6KDYC=_xrK&vywaI z&XrzR^~ji8l+5cr2D()XmFp@Mjt9vW(1dFP0&1Z-ehQ^m{|C(Z-#hK}--Nk?iDXPd z75g7PF~_O!mAFF`X4P78d@8RB+4|iKD@e-xObFSUadUllER*!G;?|Y=*Vi5~C@OS` zg0gVowF)T2RAsD*)NZ89p|JRqjF}|@$3l8?W;UAubS2D7gd7K<-d9=%l7~RMm zd3ogFPR=sHyRMTQGn1Mvdwof&p=F={#nC2s9kltn#eUS4GAWOSeZ@Pc+ z*}tAY+M!$$O^ZFS#!qR7>+C&=ORkUXOQH0N&lyJt-6GtSd;z0bwW1%chSX+hrHNO4 zDPy%ykk>0s2#r=rhz4RHmf%VW)e$E0TVHskJuh~?FXE3un!LbV*r6E+G_+`NZ(Rk| z!MrgN&$10CYi6<{u(cj-Ce-?Kf%4z3*libjm1WdjU!I9OH=;bBA(DpG#yrIUngw#; z#K#FhoU{{VNSXs19>;viHfILfsmvuefC)4Va|MTu5=h7 z^>92bT%7`#`8){nA)G}K(B0!;aK1T{`f=w{mS~~W>zNCNuKe8L%zS0e;T0ztL2iOe zRQ_Hc)LjNBQlJ|fvANoo?}bnIpEZGH(%`HE{l@(%gIZ{6ia~mH?Y%19vqM$RbTD~G zrXh+j%dh1oG^vLslmF~G(cRcZ zBsv-;bLc8dW%0XoBuYMhI;F2IC+z1jFjwgN(Sz6ia%&^u!}TvYj9NG7LW;&azY1^E zVppr3?~S>6_qi(6?JkEuP0_GAzE?zXurxk-?>_=BWQVbfI#$+f&TazIip|+#-B9AM z5~7lCVs|swX?XeQ+M}SbYbLunD}Rz1Wa5J@y}f>;Y1n>vOUN;~?1{tlGcc7kQV=fH z!yO+u0mauvo~t0mG!8#5uX-cFK6QIYnfXzx+Eq%ZPd)?6;lEMSAVy5KPBhD`#q@WG zu`Z~qkS$-qHses)A@CsT@gV0#V=M!`2&W6S#1|d};@MsjgOOzf z-<>Yqz#YLfi4Rg$ZlTw%(Q9#(Dc?CsCq5B+`z$MdHuY?>yI{(di5atcnkqYz4B3HZ zxX$3-j=oN9BJ?lh>(zaA+h+jQusIrg-qyTT6$c@uss4lAXE{XSpPFT0qhtHy`ZZC$ z*Yyz>ubsx96A@H0NjT*Dtk7tqPl*!{$sGPwF(Ls47v|fqx_<9SJ$-mq5t0iibU`cm zJwMh10=qi7A8JJzhr=FQ7p37*k;tcftimr`pkv!IeZfkKraHN^PgCyDyXO~iD0DAc zKLOfcy2tml@kGY)J1*aJr#|s7Mkv{tm6~0`JGZWMhmegBVHxlz=}V$oS2)OiL=?d~ zia?uHZhh6@=gTP)6Piir*Tex-*{dsPpc(@bxg-H3^51#?Byyl9v51Q>CV0V$i`K_Y zF@cEiv*8o5@A4I=&BJ%Xsa_!35$%xW@dFPq>3hi410fj%EPDH7b92LFygmtaa zpqu`M?Y4yvwftl6-$@kcYE8R#4?@MA2Zk%N{|9mf{nFz5cyuEfmd;v#r*k>&Rb_{r zjb<7CBv?G~Tzq$woP8CzxMDeRhbm~(A_3;*C;%$7#unKCCficQg z?1q!{)o^_TSvh4nwyF5QfDnO=H^x#-3RRTMI{C`WVWilGRkpc=uHAi|qO_xAHt=QD z;<>Sk(a#y^i_ZHNLKGK9ctvNBwtznGgvNdOIBL=5J-q1(O}IbfTV&rgy7Flrc>h%u zomN?e2QB#S7eR`05g#3t-V9{Zjd&V})uhN6RTx3IM#9;0BT7$|^Jw_G`SuhMsed8o zz!%fY-0=K_Ze8%K(MS1gdX2~_h7GDpvSxD1IQV0h zZ1LSK*hjF#B}M7e;r#>#MG8;2((&H-ihP?S-MUg4?j%-Kr!1WLD@Zn#`9H>*Nj|vau zZ~&{4kN8U3>T(eGIUB`gmgKs7pQ53hLE-F(eSz0+cd~g-bf-0sQ^1?=8@Zxmo0ImE z%NDbJv^pC_C6F@>d7*qv#7A??d%85+j7}EuX+-qXEUx2B2{uQH<(n6)!+13OVW@@g z_Sz#`-X~gmU!%5W`aLymg((6P&i~=&+(z45(>t@F*<`3T_tsO-zQSU$nm3BMA6-Z> zs0VVV&c}CyZs9ExJ89mq>5Oex_0hrDiB;uq<|XXnEi=C*nJ>-Thf_Kvey_ciEr(I}z4mDx9`D*ZDldjYx{Vyj*D(n|zJPf1F;1&ZsVvi1 zOmhFX>+G8I!}mgjTX6y|%l_Lri%mgI-~@1i6QDr&-xJ`sNw=WqMq(BYcQrOh*Ntc8 zdV<#O@uGU*9zU29I&Ehc!m^Wb5WM}r3j;X<1(IZkB;y*Kg81OGlONIzJ!tc0v!K4O za`Sl)MYQ6xz!imUYiUcO9{&6Hk#Qyjx`TSB-uHqNQ&)a!YPCst(H;b%Ont<6;EOA$ z1#8XKHHIqeV?G*@r+So<6lF8^OKK$U%(UG+_~X^HiJvT{``o+=XMhl?ZHjzkgeW;c zEf_kG&muvRd(kY~OnC43Ztd$bsjv|AKm>IUXXoE0$otrd=@=(RB%D`?3)O^glH*0Y zE)b2!s*V}JR`5xCo@XD$MU%<@5JLaJk?VDezf4H)EbnL`q5jDy!ROd+gBa8L%nYaQ zX5m{Lsyf1vaYu$+YN@F z6*Sa&aC^MynKq~gO~;UI#~)Qn!6intO1D0@4~9ysvHqvnC$M2LoeZ20gKpv}+!3kw z%4@5evW7&&o-KvTYEKs(nndXUt<_Wl^>>_PrvcY0Nn)q~COo>$IDJ`#tv1DgIi}+b z2gi%w0sB0k3%eLVY%uFUGie1EOn2F%bJs32mLryuGtDm~sZ$;%ysK29Tqw0S!zUy= z7OvJB4U9|G%!w6|0eqXaI0E3LL8y&^cF4W)HgGD{1h$Bk8dBAVhbKlj!%STk_5`kmAz9$Kd{ z5JJn)kwghKLmoduAWz9GiRV7oTx@>hVb6V6Kek?1$d6u$SIYvj^O__`_@!v1flqu< zD|(Ks`pRpMHJV3>w$_G0_X&wuQOSwE_N_?e0{FoC`eT z5fKi9$KD-cpxt~57aNy%^)B)fg@J0A~^#0%$KDPmzN5AeH!jH~2Bz@14yc`%M zDaGU_c)zy~Viva}9k&Pxay`igS38&){Ld=7t-0* zx9sI@dbRaXbK1RuU?40da5^jjvsX+9lGTD}I#LxZ`EW#L2CiqGBC;XDOV^3jdg}S% znWBr=AieTRzgUO}V(r@j&1<41N*h<_o@(89RfL19d&Pf7vep~h!H z8p?&?&!yt}+Khky!4y7pzFSf&ZIQ|yapC=@CgoMCXWa_yn1@7gi>2Y~0vaO!o#d@I zH5THZ8brcquKm;%8L}})1(m>4?syapFbH@iCnpb&n0*88<8!;qUVkAW*~C*pwg~tI zm79Ubs)!8Bk+R6GPw!}6Zx-gT*;AcS%A^mmp}COzfh;E{bq9=}E~oqyeNIQb~JiW_}Ir48dp2qxDgk@ z|6WQkU`L4)MB=6(<70jT5+MD%@5x;(%X4(TtfyQyYU&Zmy)R4|X6+38`u;7(e!8A3 z-5ruKA#%<5+=|PHY|}3Z%9)q6#Zb_2sEJMfSyS>_tAYZ8EncgurWrQa(F~B7`MF!7 zj;`BfOS!^Y?uE-wRpx;(De!#aYquCkUVQTm5ES&jCb(7i^7Gz*^sWECHaNhlpOKNd zoMYhG11RJyT>BGwHjJAMi|%RB_Up7j$kce3k1Yv6n^!ots+8f@XjxSsyu5V1`c)}y ziZFJo<#*3S1UiKL9;6lRzzCUwdNe#^uP_FDI>A(43nd zju>}mDoLp~PRJ7tmmbgYRydqHmOCZ|l_9r>k-$4UHi#2{7Ti!nI2~>Ppq%_Oe;Awl2nj_!xTzU+%-?#Pl_V-rv-Q zJ&;iyl!CnHMkQo16kvs$F2oj}D9x_jWp9f{jAoOU_sSlYy1rjCGwwXwoaYA}D!-)> z`SPhSge>91_^M?43~WUQ#}HLDMIaXUsa~I)Mf^E8xyl4YRp`&F#6ads64{-M9cG!v z_LU#StER>Mk&5Op>gBrDv)5A^(2gIC@A?GL1?J}6zbt1Q|1jRjB=@RvVYb653X}r( zXG@|!espviB=`0X>RrX;bnzSJgG`>jpUX{nLm`w-`_>cDSov5<*j4l2UkmDi!}-xG z&ytF{X_aMK50#i6^OQo-f8}PaNGpfQH*coL#w~)U%^2$wv}ak*nsYO1Dus&zsFEc8 z+8;i4nHf!GnlRf!E$Nv%>(#alv@aTSKUC6^VT=bkA;-C=H`iPsGYbX+>$9vwDyokc zPHwoW`Ut;S$(`u;vXK;ZDvn5_s2@;NAXn&tizC~5@W7bme|J^zm1C#hoCP00Anc9H zzocWowQe!}((c+8>0Nf|;hul>I5%{j_dQ$)`E(A6NyD5RxdBvrIgBev6VKDVF3$a5NergH-G6DnWdQV2!$?rV!je7g8lF*1=aSErS0NC2mOa0= zzYxmx?gKMC>uc|cbYLDN{cuu^V3satevm}9s7z1Rs$yG&`F7pW;`y_l1Nqx)?&^_W z7?#!eZin>38C48Q2(!nx#;Rk#>v!qO*Ymk%{zA;$SDWnDfkY|th+<&Zhz9tazYwM% z4LbCTeO(%7(bbZL@UAs&fqMy>cXx_uD?p);5`2yF!7&#|eb#()Ydhu`-ZylqQ;k~< zTCe+`i(rf+g%k4Pfw*@fy)&nMrzErZuQ#4~jw2Eh3y&g!xpQ(R`-N5tzLq6kb!IZ* zh>)v0{R4N*~LLQETIJ#y1Ol&pd#Bp?mv*o;{R7_)I7a^?!pD{jz?US5-Y1t>XB} z0qFL-dbkyS8ynK~-r!g^x;cMk!SkAOj(zhvSlF@Y8q>O|$9f1~m7eSpjq+pzhlv4) zVK*g@$J}J2G!_KDG;4O?9A~xu7ou7134$&j&vx8?I7qqXxU|UJeL_1( zIZT3_6uQZx*;cbmYu`RlcMiTjMdfN4TU<|FNb3=o3+T%G_!@(#$@;lx$oz{ zCYQ07ICDHTA?Hp_bL*Q+Obp>d_TIwPor`j`K91F7l*! z|D@e7+PkTKg-M4byDu3Yq~V60qh8J0d}X9@cF)SDTS#6K%IL907h-4j+iznVI3`)Q zq4(F2d$xv@?-&^j>G-)cnIWo>!nSBXFf1^ww+r|d`kNvIGNr$0&`bZ;bl&-jHuBCpq?kaCQ-EQZ2IN7B0Vg45 zs?CaJWPj@SQu=I{fG89(Jsk?c)U(uD`5oW%!dZ7(?{C2OYz8UA;G2(8>4GZC1!plF zDR=%tl=dYGHG~%`F*IDPpxiqPZ$#d2F7*GqC_Oxwy1(KK+xP45a z2VUdje0q>sHWmHfBHE3Iye+9G1D!dtCO&@krQZ=?u{pEH!EItd6*4kjOS)WMHMU1% zUSj|1*0$4zo#pfyav*w+$|VUQd$sY_Cy{07Jug#xFl<_~83H>&@`Toqaw?AxCJk@b zEtNMIt{Q0?d6xSBG56l#RR90~_z{v_GO{bA?3v6%C8QD+nWtkD|YvY`ybXNU#FSv&tL zB_WzuYwxK%eajS8)ZWCGJDEdOSrh~KI7|UnjtwYvd4YaRUjC5nKUh384Pg~G3=2)Y zQUQqQgoGzB*B>fQNv!VS>DWONPaTYHo zB3J}dBU~7BFuZAQ`)M2Ue<7E*LA7gcX4!kh#zf}W|wNbhTiK{xM3jGZx>-g7{=@AX5b=_Oif`kJ^Y z#j|!#-;L7vZ@=8@_7jIC&#;3NRS0z18WVYsBzmdJSJKM-cJ1+zg`WO>GQ+O~ee5oE ze)Gu{q=>299*8*V#|W+O>jls(*Io9^o@g**7$n~Dqbq5!46Mn;vX6bd8kPFSl`geO zFlhze1-1JN0b+C4jzC*@_3l%$dEV3GxH&f^OdDD@ zZiW5`Sa%Pwobxk8k#6ZHa!KB!YaGr`q?%hYE&V3$FQ*W5_=xcKGSoQcv8mqbYb2v# zT`E@?$GirU6kdd^UazpSYYk&Ro@P7q4}PM2{r)l(^c9SAiLX~DNP z{G0kS`=kK5SpMJCpYG)V?G{olh}z`g8hD@>g5xT>ROHp5WW%yKbJ1Va$n(xzQQ~~E z@C~sE1I?sXb`{;@c49q@DoXt2GzKcX#yCz+dRQ?RdgA;8m;?m~z_hJFlp+?e3_@>UFJ&xJmQ z?60?}+yJj_@!zjKls3UfQiU-f*Qb~NLe$F9pRSYdxz28*%n8x6xw5&NM*VKFg+*6> zKeKvFJ?@i7cdkcip&VX`raFnjEXD&XIjWC1VPo-W0ltl`88&Xet%^&?%Uns~E^8@I zlOC_Bc+}Z*Rnp+HIGKxt$|rOVeAi$;hxmaPn4?kiT>w(5h4^d-1KS+AjyeB)0&Yb4 z)&1gqz7Lz07FU-8R0XN1xD1BM4qd@|w_7)n>iGTz^!Q)7& zYb#dOKURqId9ofxm;#0d)iQ%;NCw}T&ft$2zc^R$M9HR^u0=~v|F=5@>h?tnpNLf= zEl#kpP9b4QoX$SaL$;Eh-;uL<<3Jn?}HvQ_?we(%wjq`6*CqJ zqPaFRmiZW^hxs0VA=-%m&nQE-Khju60LUQftTf`4tVh-4oQN{N9L{2*3Zm;Lw`7lr zK_Ov#ZaYNDJ)vvfPgI;^G@dzdT+U>l^L>y;#eE`?04$^cTO$U~QU)w{7c6U#Y%vq> zL9ji?r_W*zT#s8oAIr=M6YO z=djkWEQlH(OE(?$o4_2zuEw!n5uq~uIcRxILtGY=UEzgb7K;fwX@`EWMO*T{NB^^N zRS9J$@?&99WiRsxZ!|8>IGnSJvAZfl^>E^n<#EzBNDw9Z>&S_U{rkf3byJGuNDn5B zr77*JTin$8=8W^g1NFpQKrWwC;P-ab067ktrKI0G$(y{kZHXlSoOG$h1g&_Ifmw_K zYMZqhq>xgwc_#G6(9%sK-(`aT3~C@^A022hhf2&tHvZUo@H27KxMoI?8#7Ft@K3Q( zUZ1^xfEld^9g$%SfIvRL9NpxA?CV5i&KeM}dDygCOBT)f_lwpIjFfK+|Cx%ZTF$o5 zX(}U}@wLc7HcCBSR|Gp*H-s-t#CfcqD%gJwZj%CI^?7_gEI zr}FmJXwO;qi#u!QE&-|l(&o9rhz4ZYXv0D6(D%T8A zx}phoLve((jA_b;h%gyOlRKOj`?udFJH#d7sC%s4F{lSL- zmGp=E6~N!w=6B_i?+vu1GWP;bGkS5e;)q=x0aIqCEX|c??}MOVzX0D3G6b`C=D+r< z7Ar^$Kb5@w?uG#puv6)DNQM<}=vyTy>*vggrk7dcc~@NPxwWNvw6?V5i_Rj)Y3B;0 zY8>a83@_!~uu;6h*Za1CXTDk|Y7&r=d98w+@h9qp6~5~p0M$~W#R)h1E2J(l(yeuP zC|8HZU5lRg%ZUn@Wm0Z?fHKj0H7tm;WBW=;eg+wT!b`@^OU4E)$2 zaS6%bI}!(xyS&e09s${T za;)Sl1Ks`a!(5CN4^WbweePs|ZT*;F{pXGL75CT~?z7AMv)(rBg2UBG*U5j^XybC~ z%{Yg}WNM8~pVn3=T4@&kd4FS1B;uamJ&qg-(NQ6w!SPCy4PN_I}!EXNk z)WhE9Yw*qjZ_C>TRwjPGhcpf#oTcbvajVe?Ldx9%1%X%FCa>i&?86e0RS6xB zHu32869v!W+4&2pM1JrGZ6E%WG$Y8d&Fm-vL^;&nfI^16-14dBQYu|=9dEK#Z-d4A z$xAoAq!5P{$lBkM?*=}BY z&CkVMkb8&m)hl>97TUsi&U60A2TBj8cJrX2=3on-H?Nh-sGp?rwP9yp6W$Fl$R~9n zcT9cdNSi&Rcx_eQJ473uH&iXx{V$hV81IB^^Yu(sfOuIDlf4Cx$2O9FW3ca5XU3wF zx0L}CT4ax~zvtAF@5i2lN;K2Se{`NTNC9QbGGs?~3%pb-Dl&Z`pXL?MZ96oDpSR5C zV_54{GrKjrL(S3%{I%k#zfPyX5V%1s(=w8^!n;E@fLtFJKcRSb1nI_32WJ!D>Sg_} zwKX?IL=!EY1y99**R&Rs`cK<=%RQ08SP9Sm4Z>jhNCbLZr5m4Gxqyt{*>06ib&kTL z)7N*6=i7)M#4jO7t#?6?sSNQG#}fQ|zy0zn26s?~NJhYa=s5kjFo z$3Y|a)op`npZqMl7rU@+W(}eAR04!!i_Ld zZjrTjo>V8ex?Z*UC3GhLJOp6r0Si?x24peTiklZ(j;!+w*UL+cp8u-2s(AOs_EiZh z_LqQL1C&7OzMy}j-uep;m!iW}xZ}F?s_rQ7-Ruu>X;|=LG1OE#uSP065?7^khOrT8 z*_xr0Cbn@VWd%Q*vG>p3Zckr$wN>-Zx!b+;F;!a{wu1!nu#8ZxzmPg$hG_8UK*X>(0l@ZZ&@@wq3fm9& z7oA!SeOzhGdCr`*H*@pp42L3*LY&T%9?0u6HU1;UL!wSsQ)hH*BRo@-oZ`O=)i0iR z9pr$VsD1hFOO|W0Mk2>fXCVcXy}Ys~OdcHH-tq`Hi~%r?R7wQMh+Yu>LUgP8m8j<* z-fvXn{*#*Q>P-8AzO4V6)h#6+)!C7y&8@0$rjy;y&biatN=24&8lNH+0Z(rae-h+G z^$SpFjVEdq+`+lDJup5TZ_|xPg zQy4a6Fw^|-@hFZ=hRnQJ2|m|_WF;8r^`fofgnd?`9^e*ScFyss%<4}$!#LdFo3wL? zuzgXXxk7Y$`L8EpQg6STpnNZOLwpx!et3k*ai?DG2l(~2YtDxTdA<|`UhyVB>%S0# zAz4i{B0FHbxqR&|oL%VGqkCGNC`6GbA#2}JeKZmClYjqLAxRcY0NKrR8^O5Oyp=|F zyK=N&96Y3muJnq)<-SY}rPJW{>`_zYW@{E7OE@;c#pHck@t*p6vtmHDOZ|0$DZjf& zJquS=<<+6AF+82!N9=OPb%4UaMIu)bopqV^cNyW0F`ugmR+muXeCvo zCGA76GH@r#1*J z;etTY83*oT(_&`MWD~;Yh5Zm(Lsv?E?%chY)fpFTpm=|iohm-*Fl{z8LbgzD9vz~d zo%1@=amip`I^3F5m$32j5I_ZXF8HiC??0` z1P%v?wHynslgtn6uJ*t6;*4m!0)krMH4OENJBb}Tocv|1CeC=Y{ySes^e@PoGb~9G zu8^YlwRlY~5*Ue>(v}x`ff!v{Fh|yO8Fr${J0VLlb!VwnUf_KIZ82AT1Y~WkF4K2>UGCV;L`pk(P6b9S^#5dhgDmMD2_o7d=IL#oS5(7gEf6fN3d$GI!d?x}6>FGLid zyq86ntKqlLuYMWv-1U9$a_sthGRlUhY=4zKlV^|2;C^xBg8`8Z$F56`FE!5>cZ{c# zrTNkTC3Lb+^L`wR=LsHqeOuFaE9xqIyf~zHDPO4ZDIcQW=bXgvZ!75VH5Z4l|EIIrydq}I@afs(mI|NOEP*%;JiHDk9$hAi zxHG)zxVgP3$E5CFU?^a&4SD(k?L3p>-f?7&Q6?#QCL0c_2oqw($~Y5@wL2UyKXb|F z(p(fem3jjJo+QY=C?o#ypn*Y|qbux(qr@%g!f()|u9d71GG!@mx8*E9N7%=4-iWep zM*XtEy;{9B+6kK(=G2!CysrS6`DGy56LBK{UzGfeAAI4*Vl!)I*9XN*mEHS=IrEIH zw6k2VmT<-HL|uq)(VB&h0hqW3&N?{{?)~TMu)C{hTcUjo_h(HZsV0ae-N@kJW2$gp zKZ2lmbD@FGP-*ugVbR;uWbmt#XDmG5NZ+``PDfi&OjRzw6L>60Kq(PIHUI5mj+r7L z4Ord~#~SWAZfVsYNrtn!{d7{>psfu>| zQFArrzh@a87Rr2uPx5k+J^M2!Bm@rdj~hH*>GYOpXz`6N@bBbEwoeq)#L>sC{^lpe zH=b+C@)0)}KY2t3myk#vhkRso6Pv_VE(WrfsgkSx zr;n;` zNP8FnnV*cDE*ZKQW8vqaH&ny;ZG6ojZnDYZb@4aJ8lA3sY-F{~L=JK;sgK~NH zmuMfc@pI#%1=I*6OLkDEbv7iM+fTzX+Bc3CR~rxbsxL^!A;U^h^?Q%MYgKVdHmu1C04tJLxO~ z;aj&|#%APL(s477^pNalHj}cTna7{EQiZ>ockmV4Kyv@McpiaDaKcH9@11-t+!>IZ zB#^vbuNIii&BazQN*Y4`ah)tnGaSAT3+-XdL|uIC{neJD3}-u<@Kc1Fu+Vk#7zpu^ z)?zQ!cTZ#d^FdK1H4E3mb-armwv+#ZD9hXE7*)y;s#{BS-_hOTXFb!5`-so7rdP*5 z4;n~LUe@2GrMnn(4upS~-*P~OzWBgH*L781 zgFaXSXtpJV0%Vjaa%LFR+>eHJIN-~8=)>1*AeDrFIKJ_7;nxj(CL7E&#c6#;eknjD z{tHHoAt@9(5KtR)zqY6~K*}W;Eb0r|ZN%Rfp1r)mNJ^N&FXh8Y|fMy8cI#i2bEzFcf5n-vyW1ES_ixF}NKh}72WP&^TM%jqYhNEov4|-pn;?zyoWn7O z<@CBeB-gGQOfQR`O;XexH+))hhM|Sx!z@6*cVn}} zm%czC-8YRp1G#8m0&EzglF-ej+E4py)a)D*6Z-tV$rnET!aVuyX`W!pJJ+;7W;j(| zGN7ONazPsvW<=gSSF7JTJ-#EHXH#1mcW#K!&0<$7K<5&~1t|^l|7UGY*ji_}JFvU; z@#{B-^IhHh3u|bW&lT0rB@fc^DE3nMsQZE!rB_>+Ih@M?C}ZK%D&IDy1l3{;DxBVC z1xein5^DFtB)5*o^V`9E^zV`^TaTm9AsKZ3 zAgI}WFWH*gIj1J%A|>5G6%)#%<%k_KR6yqBJ1XxZ z=)z1}v};4^EhHnsrSq3xoD4j?sJCND`dcLp*(C_CZ~O$;;Z((@*23G$@FGze!xsjW z3(vW=#k?Tfx7xSgoK9A8K1jQZH4FSYeW`mWBn<(-5fzu!FZ(1&b>T9;u(}&NoX_(U z^emr;2u6tB?iVhLZWjrA7_jJ8 zL+R#aL{xiRBM8&m_)Of-$@{~@Ng|I+;yGj(TG*7%ctp*5)Th%TOQ@PKETcr-(%~{w zCLk{Lec;Rc_5-XE<*(Rkq4XEDfP3~8WO0YA^CVq`gHQ)(>3(Mbtd+Lzk3s(9yVB2% zmDWoGBqr3xo45g^O+5_7u^g@Bl*)?}`26YE#A$ffh)*z~y!?lRW`Y4;NlG z8R40WzBfnq{BG!O7Zu_gize;eSXPVBs2LDR=qO}1YwYdd zE6MczR?h^v6F(3;lH%e*8dOmfb2naz04VKGzoEFJeau+Tf*Be|)y#Gz7*;5nu%zhf zMu*SUb@!41DvK)+fB#|pssUc!c)_zb$n0A$!N^7 z<1YjilKeuMcGJNk+q1{}(AM`Bu4;95n+ZAl14+7uxo?}!_~|dil9AjqvMJGMD<8d@ z$l<^yDNv(3)!KLeS7DsH1QkLVUazEu&1qB6|LJE*VjC8Vl8Rj(8|zY%B5${H(>r$= z<=h*f#&RSMV+Vms2F?QRff+65Fs)1C7pCyIAQMN)W%W^lUD=b%hHF2a!FYD0iI-gq zC)ZTLrouD0A?kp5S)1x}Y$&=w&6$6m_IelB3$h-SfY>SjVO~a}?l%I79)7P6?WV}S zsdwJtrQH;}^ykj>ck;7)`MjXvY`e__H2o651O_aSB}EeysT`v zq$wObFEOYpuV!TXNZ&y#F7L@ldq2WP*l3-V_9xURN`QZEPHL(P~o+=UPygT82=hEDu-BMDms*!uG_u*Q%kR1+NJkdxuFwNozM_ z7hjvRSH6HKhgQ@O*hQw&0`auUEF2+MTC z`G+3sdHxVJ8rsjNms)tlphVdGV!L=_2F?nSFYf$J8c=eqLWQRa!zeu2^l`LC$t9K% z`sXJ+N))~<;@MxOIrLKLsQcbU+B5PXd$=z`_tyXf&(w!)V*!`^j1Zb{?+pzT+0@%P zr!vBf|E5cG_SyALt!0sO^80&>LP_M+!-bprOUvF3G?I;>4yP{i zH5!CgbGbEMe<4aTP$DIgBo?5ofmgxmW9E`|&abIl9&nOb#7MMJ$Z1-F z3~=#0qdk3P3(%5^mgMcQaqI6%Qw0}btHIn~D=78u`i&lH@n879oH}2lV|bFMWO^;% zFKvH6tv@f9uBH zeh2)kPZU<;P6N5m9M$HW=R6HnQZYzgyp~=b&3?1U+LEm_{pa0J`)x|4$Ki+kD^&QvdXO+e0{wz_!sM; zyWcx6$3Ib4Rmuvn6J>rE%&d-8eeYBfTxToG=vBrQBaia~9jTHJ&TonshU!VwUMHW2 zf>o9St1Jglp#NIsLZl`7+`2z2);jAN#zORM;=$GD>7Vc4XEAUx)a+c`qe{Z*5Wwbx zi2OsyDX`b)IGdR}y2en_BpUtmH)NcRzLdK#ws$0L6-le) zmb-Qn?@Z7(30{5`q@B!e9xgrg`8_51;7Bde?j%D=Z4Si)vo+)TQLHI)F2Yc3L5dv) ziIZp|ect9|_jbPnU7{9WI&ch#uVOlVx#y+SS`8k4MAZj3po?oR;Jo%x zK}n&}h&>#>P(sPzeDkQG<>MUT)PQX+u7%XHSv~(j6$L5Dm>Kn_f&q>xv&fm(WP_v* ze+W(mwcxXfFzN&Iy_YtoNBzoKgBRT``Ru<_1TZh|0ejzq?P`4t#?!d)B(5RxLNp?$ zHpqw5P%t_(?w-}*hewxeD0tL(J_>?t#0D8%-G-{s@cQBz(cfotp7WzknO+2oCSAeT zl(dgr%fpr1;h0in_-hn5oWme=qjaU9^f+8(*X^R_7Oq&|Ap~+pgSkF_F znGebVABc@XDZN68j*kuhg?ryzv=zmu$k)8-k`ADtnp;oe8`*!zXwX}l|M~v#CHm5E zJ5FsON%Q*mj~`8Xa33v1?bNZH#NP&&A}7|;>~2mRf&d~-(b^j^(W+n* zb%4$gjwPVycV}T_Yu-l_e&38=WoC@L zei{fTXBOFy;}c!8CwgcmniHK%^-9i8K5B(*9w_^Gpp^mOEgiQ_#P)*z5J0yOs_`>p zP4lE29EYQU&PbtXW^IZ3XKT=@X@3Mbm{Z=&$dK)9N2;(879QewGHj_E|6PBRslxhQ6 zh}IrO%qihhDL{b_G|R1jd2aX0ZuN$y-V$q^Y9{Jq2swX{w{wbqMHBD1Msqcw(eM*& zW>t7yZWP_@{$jkfuLi+Kso!CKj$%joI{CWM|6)1@C1>z&=uMI&pI{5>Qm9_evC+zG z)GI4<#zpvt(tB}R%m65*5d4gte>ol94`l+>YpC!tzFhb?W%TB%8hg^+!-$xG*x~d( ze>Q^P-ND(x*BE*`{K5B|gsi6>_gbZ-ymwW(-vOjKc^A~4*#k@gzT_z3GEN~&rTld_ z4Y1mS4%{`&Q?V-%^VC-=qXk0Luj8si^z|gvsB-zrUfIlUH*F#3;|3lfN1+Rk#*7X^Mq_2YIBkbsYsTT7vOiY+*t9G*AEL>lG`%9TDp2?kLOD{W4X`Q%Ppe z(9O3|e<9~wzCD_{&Gbrv4l293Z~WwY>5$BEB=@~(#bm{|l3mPigw}jGAjDGiH$X`A z{)Lk5pGQr22~I91B_e9j-)+-kcDZS8pfL$?fNA|g#}I~gYQ zZFwr$0kWq~lpu^Rx(AD{?z6fqqu-*%tr7Qz4@b3wzKpI`k@swy^G_@sd+xcc#W9@6 zY!#$-rXAn@O4Y^C`Dd<~-zzEV-EO&g!FtVoP6HjP9*1?I3rYz>`qHtx=lS^qM*_TG z%aHBKHDqSjz$kK1yIgnpxp*_%hh(zP?hU zx*U-ZAdiaB-0_vfdEYD88dskGmfOMKNPrkDeWGAG*T8a=V2ih!mkuWw zh%IVF70U}kW*<|vBYz~7_&W97&%0t?#UI$~#UM}AMAxpTU~__$Was3I1h7ldb~#nC3n^Dj!3-JWrJQxd&rmW96cvAC*~|ItXsfK{|W_`(O4g;&o$gn-Sy z3iKu5Gf?w=3*3{(FLG@Atxos1*y514vo0wfRKc2-T)O9!guU1xeaN~M$ zgB<8N^tfF8IKILzo6r8Ho(U!)-t2bxd5d0F2Iu%zXH*u%N0hyYZQi3roSt|qX5iT0 zZn3Hwus)YP-r3+b2~wp^mC_N+e&t+`Y|`_*Nb=nWan@NBz??m%s_8QsoR-IKn4RMh zQ%Dh1v-SN`_2l#9f-Sel)}j0URGZNbhF}jp!;p{!(1PR|K$7J;`80fI_>eiA^G++nt1VKUQ%Nv_sYp6CEgAwRV|S#&UodU#(uTH`5p zI?dJR)kM~(cR|aSIvUnnAikW~aKk!b&s$d$_zcQ!ktv%ABzVm6(yiTj z+*?Y^Dcl<(DPy-wsjMoP>xx@EQ0ALRz6G z{5IAs!*u~Bu;<0r;p4?DS!vN8%|`gRz&Aj#2+@QPN6UszQjHmxI(UU9DBb+ucJ(*Rcy_1$EqTV0hE<&XVM|o_d84jJ^8M<#4O!hVtH~ zG+T90pZf#XE-_0k-t?lL*|cbXW_y)uNo@@ZP`nsg={c-awYYRmUg)i~i)7rbc5Ylf z)z4D{NF$s;2w&)CoFj$Fil2Y97DF ze5#wnqd=m*k2^e$TJH*7VLtB5Jz1z`FZrXa6O58#HqCEQI?-*i9;Oxy6*g> zLazD3t(8}jI#|>u<1FeT9AABxK6lK-^k)0l+F+)e{#}*w4a#Rddzi^Km<`Briu0Hd zf3DgTqm6+skJq#UgvC zFVFjL@#RNnUMVmp)v=-bbuM~VG(NDX) zSO$!uMtj!JqbZSgmezLf7;dLk`NOu^1!GzhN6vkc zHO9zQ76CNV+6sX0#;xPRBs#e9AK@U+`;~lBLTiVI7}mF4_g-c zZK=+Nsk!H`S6!}W+Lrb@g%OC43lqx9C`>OIOp!Ekt{Z!N*Gli1o5b_Lfdif4y09X} z`MRn(N2-RAC!SMRlkdL|BmL^+&T-1nb&>My(T%bWfXV^T``8b5&RIlrZP;Q5#o9++ z&y!CfRE5$cR>peqqYK+mEIpL42~dy?C&)%45IbItuaDJ$XE(NR^j_8o+)Ysk@=|t< z2dFIKBKrAB0!#8jT4+Sx75&9x^7Lb+H-b36UG#USx(kG}YfJEOqHwm-fTHH+pV$OJ z{#hP*S@awNkTEyL_sE6H7>lad4f4m`{%HN!HR{NwVxS+WEuDPPO+m?=gLW}k}a#8z;dNk zJyc?hk#7h3!50I|`B4-(hoGx&FHdq0wIHMWN*Qshei^l+rQCu3XhV2|rR(973))%~ z2%jw1BzGMo2c(QJyzD(hB%5gI2UDpRk8ZJhg4vM$B@9bn{5&d(JWtOpvMETi;Zqni z7X3qof>1G6S06NMZb>u|?qeKVzU^YsL?&>{t!%K>CN*WMN!Ev4jUeVn`pA?GJHVjj2n_RT`W zU=+wX(E$TZ?7AJ>=Cx6AF~?ZG@b=4AZ=)-erK9Nfx#6qc5aAA_{d*e%U(tqMvvSp} z4dl_=k;112?!D7SU5*}14-rD|ZAwUj^=;Y| z3qdM03p7-Po1oF;owUZm6KG-OovUjdtzt!0M7^2>SaRa!!206#$hFQky8SO$!X8Xdy_!(WUh#l z^tKob55PzRs=WHTFaTO?GLbu>2C&Vun3HT20UiR*lL&Hl1Y`!iR739#I+55BdSHUU zr!;_ao?Rx1V-B;T%}bC4Jf{GhZ^g2cMwD#=syR?BKm(5h9~M%HIAUre(Ibx^Li`NX zeSzhoe>JlwfFg4Ih2WE*{};L#)4AecLP21zfO=kSHA#KJb$+v4h;qUcp3IxQx zB6|l>JIdwR-&$*UFZJ$eRFGotnLYqWz-nw7fWd&DCVwGC?lVVMgxmAC7^5-dr7%uH zf~Z@~-D>)czKzP=Os22L5SmWfqJAnWcB&Y%4kdcjM#GodKh<5hv$SDR5ggZnhfzy? zrjs-(n0`wM5AvZER50(3Hj(=;B9aLkTw4fvRsaLftUP~sfs@KP!I=r|J0Pq6Hxnh|3Iw})#?k>4gf%at$9 z$X6!xd+IxAHYZqJQ@?!ix5O*g-x8G2v;N)Q^oijGV%xqJ423K;sff|pKy$nMJ7?#F z&r6ZrVsCRE25s>PI7wrZmei1T{A_*s{xKjLw`uqco%c;(bIjAuT62#uXQO3HL9=R& zKXKUs(v}C8m!rIFh*Ap3-nAJ5=wWRRlHETbJ@9!1)*Ynb+S7F}(0y=p@H8(nn0EHT zq{dfuhQW^=pI9HZ>d2(N-4+O2{VhSoGK<0fGJ1525N!nt%iu9>D`$8|BY{K#Gjb#a zcEpZGhu+^o2d2@;q=o8_jIb&c4~rwtX^DNfeOawtHSnX2QhZuxt&rdS#_1<4Et>^; z=%-%KG>O#dw6CAR;m3+OH{c!!>A<++_Dg~54f!$XxP@H~khjGI*> zSMu`jkl3$4SKJUpkm}>t@QiOU)TQXqRJK8xtm;JFG`;rPDmO`LHo9-$0zRh;^DF{< zoGJvWEIa_U#vF>0*uYx-09A%?;o*J6SmbSv4vizpMC zbrORwRi6^?eDjD-_!HBgWl!*)b78lOrO#v~Gu_;xhJZnE&w8r5zGqwLLQ$H?pShPs z5AOb;>=}?{_AdV{63JzNkJ)(jt>gy(5m^%CuniBQt9OFI(Dixb38nC&_sIpq5O!Ic z5IL7y=q8LG5G;-FuBbikx^Rn*U5hO6(iVer&tqRf5_=>-JAOf(ZWj1U+X{L4Nd$E9 zBPbl>FU14tmL`~*0Sh*=KFOwOhN|W3s+^9efUB$W0Pp^e1mrv9T_#OB#GB#kV(RC8 zKUwbBeOg(oTbF;Qzd7usToa--?t*^@aw+VMCkq!t+D4cs3U7Dzt0o$_W2-oWI)w80 zMksnIlZ3usk1AFGt0{GIy3c$8c?c2mX+F0>J9HeEkaj1LwrD z)En=(voc@R^F9EC%k8*OVs!P)$ti?4lU@gVOwyR4M~*(40Aospe!2_1W3G>eYqb3G z?9BKQA68Vq>va5Da(u5!ubL_T5q)hThYXaK4B>Q3kgD#rc^<@-yDVk1Un0G zQ0C5GVi-EQh_#p#`;2XSfC=FobK_jc^V(Lu-U(FyJw#o(z=@LQkKZG2_q}s0N$@Mn zFL}A(`MS&cU8%e4L;?C7w5hW5E9ybi0(*}h*&G z^K2Y3N^H4N)NOM-DhiG*)1%+bn$3Z$ZMS+dj;;pK9M0fJ0i2lp4j7L32)<6wppT^q zK;{`f;TRirvh5Qc&3oGgD}Hi?nVgY&`tH(;cWQiSK#jGVP=`4Q!z@0-iJ$nlfpfVI za^#Sg{m7NfG0<3NLgJ1N&~JDdWqLyWS>fDL`(^A-mb*pb3L=HQj*m4TQ*rAI2Pm$c zrgTDO-TPVjl+{&1XC?#r1GR{&vG&!z{UXk=aiyq#?#(e>FJ&kAQ4pDd#w#%v1FrnP zS*_hh))>S06vdec{@@G^v5a@(`Is{};V%xTinn26UB?o3D)0?Lq-}oXoKXig*PdVx z7m1myi`0tQ(cgk@u*s1^xxLa_P=`YPQeX|bmq{q^W5!vJhCtq3foa`^yZ8#ZC_-GAxm(>xr_e$;bLnKB2+DY z%QyUHkbqEia0n7Z3c!7bo}4pSQ})FG129(7gijUtUR{SvF{(MQDRA?#>J>Zpe>sM; zucV`P6h;RBnI?qjK82Q-VveW{ku&=M>;jNYlftA2#u$40;HZxs6?mzbr+xiG0O)AQs>^#+?fL-Y(oe z*#|ipiz<(+y0;whhIrV_p3pSF*ysXi#nb;ce#*nBtr5n(a^!GugbG1?DGhLnZ5mcgw|$)r$%7jB z9&j1sdu02jpjOFC<3F4f{immu{$9V&C(E+Z5Xzuyhy;l~zME7Iaaw5h5W_2<1cJxq zbwOdTK~J87r?oEmBujFz@juhk8LR5J%uMW|jK1PNe!>quT(gY2WeU7JY@e3!x@gJv z>qFm^hW7djWUOAi|Dz*kw86Ijdc7-`{a2Jl#OS|*8;mm2_UG}>nJ5)L?XK`^#&FcN z^UGIdATKr2moT^U7sO*g*ji(g>#IFfW_nTE?@`y|_|0=zDU>)L=W|IeoGK4l&h+H! zNUzy*9f8_G1|E3}$ATz@^YHzPkKE&R$~^==ebLj}(qzG2i^uZKv*5EF;Wqv+P=63ZxRg%)pJ<#oL8!_#U4vd!(h=Qnu3*y0oxEMh z$MlTp-ZR}52t*$iR7P7U-BPVS^mfPCL}2ixWpo&y2pO^AYOV2nVgoN%1!{ldv+%um zy&p9{6Si*5x{))o03R_+i!~~ZK>-M7$}T$i_=aHH4Xdb!Sgl+7pThX zMWa9-p}?Tf_s)fEnVIYltQ4Q&k>krgZ2i45Zd%{d+$Q?_eVK>?=`9A~S-vevs6U}Dy ziN=FOEjXU_=iMM-n-`XckF>cg$$kqrDXC*Q+wcm1Hp{ZZ#WI}Yc*WMHMl8R1QCJ2k z>}|NJUgD*2qpek(ZwoeSo!8`Wyf;04J31Oy6H$;ec>bY(^hk5sITdQcG-KE}3m*OT zRM&b+!$)MG`rTb8PV->?gj;Rv<8>4NIyxmvI!X!CoI^H&oex>Q7DXxKQ}A;Pd^4I# zwpWlniMw^LDoFDVPr#4ZwBZ1#gxHof*n*`kgCwu#HF1hopHN`_IpbJjW@?;h-J4Nf znQ*-*Ni$pBK_ud;J6Wi|s;sdtfd$7jeB|Ryh)#B&Yu04`*a(L>FE$ z;~~Z1xoLc*F+F_JicE$d7vp@AjuCUbjVB0A0^>X*y?ra^!zaeacO#-aym%~caD=uY z;{|!jb}cdFb$l8!VRe_P-tn4j&IR3 zm#?VD@JPaD?zBBGD^U;O#GlgSCt71Xp~!dd%2QAh7>8Qxyqy-oE~Ao7ajdDYCye19 z(R%uN{UuSpKec8(x?M2rbu@(W_4!9txwc2zYwmIfj9y&5QUMvGpWjMyp#Z@ZLw^}( zx1eqXWIjpuNrCg_glppsX109+%0CJ;7R*p4Hmy2zri8A!?5gzD zANLXybu+>`8WXfw>IKW)7nDo>KknW$s>!YG77l_G=^(u-ASxh=Af14PA|e)~3IPFW z(xeK3Pz32!P!TDD6agb0DT(wZy#(om-V>w*NV3mj?|sgB-e-J2&hPIB!!a0`akIGB zRpvFXIn{;IsZp_{NaPl#ZG0)?ir6vz>XHZbc0z|}>@8~ygQimtdQOL^*$bWHtnWL< zgOv7U3rMJBHZ!vkv|uB9Xtv)7yWCW?brc;p5C1zY=|RDC0pJbL{nuup+ z{zb>ZSF3wX<>;%I+IeItHcv>gfdS+}zp$z8HYiKGkGp80^TlwE=N+*(F8vhvaiM;^ zZ42A9!&i4HH)M&6nZLA(huc#a)1N6wW+e1ICcOK)b|jd3l#-g&WaG=Dq5t+&F&#ht z>4gtCVAHde?dmN3Ca)sS=*&>h`4=%h*5N0DS_cC*uPXfM#GVbRu$Et2(|Mp%RB5JH zeIw25OZnC3;i55#{d4L?vJ@@E7ii$9ebzT)rwO>OJ}ZBg)aKTuLkTPm58F0+Ti++? zbc*gF2how!LN$2(Ah{m4F75dr4OR9z4r(hdw0(;(MjhNO?B$LQ6>0AY9ln(L#seX+ zv;G&PG&S3YtL5W5RZW74etxH9!q%b5NI!LZ0Ruwm4>YDWY&r-3xYaelU38l>-{Tb5 zi_KUu+dDw|j#^>{ed1()35Skx_3C3Y{-kue(h0DOPA^Dy{pJ=AuHO5`jO}Zb28Qxn zu=>G@S8@`3$DOXvGN>Ow5`R}EXbZ>8_R}N~G|Aid@M`aT@+WnjJve^Zip73!UZOl? zhviq|^j`z?@o2qe{Dn`NXl=mv4;CO>>LnH^iq0nac_JGMAV?`v-X49!^;uMdTBf^L z{V%JCdgsu}#V1yoZrVhr;i6E&{G8b~FI;A1SMO2|S2^uMjJ6whPSIxHi_La0ApWEp z;D;H4m5PxDF!guxKSl;NRdrr{UEL+*%NMlQ&anu?lF+bjYYmF`_o z?90NA)chca~~lup*D>(ljt4R!lauv6YDMDo=K zx4)le@@ux{X{rAzx~f(R}oC(yhbC-={= z&DDmR$Pu5uktsy!P#mf^9ZZO{pVb10NI$_Q#P+l*G=i%De3_hR8-`Z>!BMnki z+YDmjX5&WYn}VSFa1n4+O&W9Nd?r^9X&#h?Z+A`Ab^j4Dnn-7u_r4ySHu}wJE~R20 zY|=|O`7k6rZO$Fnl6+KX6x(v~%y*R#+D7VHQ%E=7qJ$JY7hr9Cm6YW$>S2DUTy>aX z$~BSBZ^zxs2v_t!zD2y7w&q}k$*I=zwhdS2dzl#G`(!fgyV6UYy$h%z!Ut#7<>q*f z3DaAm2iG0s-CNJzHCgbW5Bf?YngkfUOQFC|p%74Mps^NL@bWDe>!H}_f=XqlmGt&= z;R+U0IhyczDRoPA;m{2YZefxM>@>M^f2aqK)Jf&GNm4jh^UB;Hk@vm1hV__jQ0JEV zHq7szj_PS2kS^Ap6FTLjv{_mERNt0*H}2-mR8fj_kC^o1VDS^6{rdHTu|1p9De|1+T!D^2NWdOvo~;aOQ4R;aSu+gj)8x21>NMvt=GbQDDK zF_z0s=(#MiIHJD`iaP}`mspI)vGc?|6{jq7ZX3?c{P&}e@~x~ylCpL?7?@OB;ee)< z3!_yrai2%LQId|3cE(o!sEchWZPC$N717l06K4C+uFxyvp0@F~Hocj{rWy_5neuy^ z@SDjU0KLCk*%DI{I#x&}I2(E*PJcr1fVUs(5`h5hYZWWkEP@4ARrP$e$zlzwDDo0~ z&TZ{xaNK129x`77AQ}^#)4pWM+?`wJHRN7fJdMefNH9}S1^i`oZg=2($S(3hp#FB* zQwHyHy2@bcKUq8vE=0Nm({sRngn$*Ze{)(y_4+Ed*@n%uFkA-S)C7)C*w^a zd(7kP^RCfERXpz>bzIM+pKV@7Vo^JD8-8fmp>Rg%nN=aFHmm z_Oc7n<>s)3?BsquMX{Qokrz1ZJ=BP=gC*J9#3_zWYkwFHJ|7`s-sKlSvJWl7b!Pq;?W$H@n?uCxUKBDv78DthQo#(C3d5`^l4D>)t0R`?Y8S} z+9J!Y1bb-Y%tlyaz$q1H~#bc+<%lnv|{I zoSJuTQjZv(el^u8yi0VNc3Rk;2%q*S@HJexb*{=dKc>54gW)dAkJa>3?wjg@0JCMJ z!2_>d7%}1R-()v69`Et^-M&vPet7@J;M!=>0(Up-*`!xRED6EAGL2CoN|vnY@nP$0 z0qplt&{B&MmhvAm%rS1=T{n zym)n`9&Tno7{8DqKd@QZqpA8>zqUr#L27!O8+Ckol81#w(__-#D1opx?hh8VzE3qv zvPwHgDlQ0RIHf`7Ybn z(7xD0_en$AMoUs%xR1t3Xk&bxJiOr9IEB5n|C*3a3&kacZtl~&lI6{PHcOn9h&qE; z(s=HauU=xi_h7@$EgR(}k*c|Mm@3HR!2IUNS5tDIq1uhI@~St6XnF{b6u%6o zjSR$ZHv61@L8rnp81tRHi0ECJcp?OlE9bq|1Z9ShkN4$pL}$B~U`N`cBk#g#5t)u# ziE(GcA7`Y;n;rt;nHkbMHE|~t^!sRu%DsWfsntAjv(`H?9c8YECI+YH?HjG6eig2x(J>=Jwe^e{QNx5*uS0WUr!ex4X5rV7@$t|Z8rFRcoq+Uj`_c6$yGJsGBD9j!iHKy## zZ|FJ8S}07`!O`hfWrg2Ua@52A3JU_sN@<4Qs~darySf z6O&qo#EHzmAWz_W;+u5%Yq`&1tY1%gR2Z9C`ZKg=_M#dItFCf0P0+KA&J4-F_Vub1 zhRgX`d@0A#iUq)a;&=Q!O_Yi)wtI@ERi4!mG9fo?bY3)ae%yeg=^~ZD@72Vpqi3iK zq!^GcqkK*_`KU}^L<2C{KytN$y`Q~e9z!#T`uz`*QIi=%zEX>mZ;myy7@Cc8syQX9 zd+G3&@@obyxJ$1xw$5=Uemrj62^nI#V|Gilw}dLp`ccc}Utu5PexB0VI1Pc9{(Ya7 zGYN%SWQZ98UweT5?SVx`z`fjy9Re?=N_1|Dm_>YRabR~Aceq(_DshsZ5~H|@_X?0Y zW}N^$?!AEY`8Oc`AD!cw|Hf$p3Sk`NarA%a_yD%U z`UDz`n-3))0$3c3D1e=4EN}rrfHF48*k&f%?FO=0o7 z(YjL>$B&2hfEfPskR9)L+ut8_87I+-3aYMe9TEEoU~uxEvU7Prig-DGlNEe7|3pC% zeI(_ytAtiT*vB7Asa*yGn-nn8a01vg%z)f1qU zPtb$fIq`BPmK?$+l%9lVFD!Q-o7d`=EA+}!#?0ESNT>nLP|nJlD1R8^Ng)_ z^M;x6>#2c*}00Clf~d#MZMz?jM6_G;I0wlk4ZQyZHvo{aB_^W2!@au)}z zS6?KSN1tl;s_3p#iWT%rW>P05%_tS;Fh2);cqzvf7j2gJr7+^Y`#Cw<; zq&80w!MipFUc@uDMQ0JbrI}%U+=-kD)*4>luL-}UOyLnxjU?I;l8-f31kd1Q$AyQ! z?MZwLyu!$)GKsi!_7!f{xJz=rEiM$fqtg0G-#{V!31svc=$yDVdG53 zsWtEY10a=c>nalFAD;MBlldhuyy(X(b@S@*FvL+_5@;1ZOZ5Z32@<3THGS zt(R1VSP1w(e%YQt6r+#UH~K!_F1%Q-f%BGF7?v}pY*U1)=#oH$rN)tA>no#;A3$zW zq8Yls2gDg4VZQeNEm3J9hi}Ked#xf~V6v)quKyYS^wx}WM-um>dVbJvH_iLXenw@O zW*Erx#8f)ydJ{7#El63yH6F@1xOPE^(fQov*~21d*SE{t4AKurQCUN-vfJzgKfVGg zpEQeXh-F!xq4Aift*Qo1g3}HWv$FgX+-*?{Hqfwo!n%TF0qKMi7f(7w7mT*rqt3X0%01}b2TcjnfQb4e3gRN7b0gxpOE zdP%LXwuhJUv)dQ}R}Kx<%l1ZLBvv3sEP^k>E{DM7ur>o;eZaXx;O6*Mn;9w>sp_D{ zm$N&XbnVr1EvrI;DBk9;zdO*(&Ko9TkFLyDR8KKO;m%JNyynw((BuAO-OR|pAk)?V zXw?A)MtST^i!_6;QDf@C_-2{osNZ{7xyk$Hl8!?&1*;gyEssqLFqfsDnM0D;>7hhw zaIvrsP)FB8F*E~ueGIV@QM7ayLJ}c7oM&Wjn`WvShAToUDPO>X%wV=`PW3SNn&^Yi?YtMC+~Gb$qfTRW|8tNcnxIY;0pqeN z5KKb|;B~|ddVW&Yta1tv(XaSkZ6_mZ_`Ue9agaPq7Q-w>rj?O>7DIr3)3nftOWL{t zSk#zEh48q%m{05wpf+aLd^Zn&-LetX07n-b6QA`!_;56?~0cAg7rS49YcrwQA}&5$F*-jOLm3gd`8c0zPM z9z<`@bCKgSP>7QY!n`(a7Y&zlE_7IQsVKS0W}uxelU^ioiKxNxn9QP23shW7vOxSt z36U@Si2q$j){Y0B=P{W_itB>~r@np2H*qkGcFHrEFc~itv#O4?R@WCllXW6q@;5oh zhy(A@mL?RUA|iJXDHXO^vAH1jK^<&kKBF7cnCGVYvua)fUq5z6bOZzG?|X=q{4Azh zWG*xiX}rLRAkc#g-h!zkN)W>4GT8~70Y;j^vL#SGqhmc}>RHpU)A47n#n|9spDK5k zH^5X{R30!c{6Rp-?Nb-fo7ptWy5lA5=8Jfk#_^I3CdcI$+H75=rSJ_sXVhLogQbax zixfy1{fIcvRc-QKA=6&+Cu@Hg&vF6HP-dva2ms*@uVnBYJq04svK17&)h+088V#IQ z4Oqp~V zet3Kv(dmFUBEf++;@|pf5_ltQYBX+Lqik4u)a-|AoRn1d+!xBzAAzTu)nU|5Cc#OH zogDT_mkeT)2dqk-7^KFJz*P2;gPmbA{WL4f;bd`Ne{;es=@^j@b3@Jpv4cuUIHe#H zS^<3#?)5~A)AH@{;U%)KiLw>RlK1~OyQUsMrLNDtA*OMG$?gd|g_`#;E=Ube;KFM) z30|#34c04W=Y{%4?}xrf_)PcKlB)|MBSWJ$vf+j2+He3N!DXJ@=z&;q$4pfG{Re)e z9ew_+7elUPhzz6DS&ANjCeh6aBCx6bul|A*z@bFF7Y?QtnYm^GAMch{mR3u`x~d|r z&D#r`g0BB8diy`8+9y-)|1N%zl@eI=gc12}D#-Mf_UK*sF2x_xe-%GSR-_9l9;Dn- zEua#~{ugAUg5p3<3ur|M4*o4WsPhnHxAqG3mJsm1NZI`lnFh~HM#RErf)Tl?3Vo#L zwA+NfP6m{_&O12dza9juVGxeP`Tz6hLjsLV5)-mdXk3J>s~?44UqtMh_S2+e{3}nGF&-CcO3bLF zT>~2Zd$Cz;LVAmW>e_>>ml|#NeOHG^U=f++AXWiIQN_xl=#M0|E%)O&$L=akGM*tn zKota&Z%B@bsCYV~vTzeeSMcc7Q72g&lH-nW9v3?0WCYgW@98*4%7FpHwND!T~ zj_bS;_I1qhq2`3Z#ul5cVVm3CEHRAA<}S0>BKQNUr(L!TAIS!q^>dTGh zc&YwEN)C4Q?kIEGJ<}0Q?i4-9Vji60Y9;ewT*fhXFuSpuiw`=iDjMSK&d{E&Dq@dG z?5ja45$167KP|)?N=H;+C0beLDOXE6YonM|wUqTgG`9JB^uPmfFCCo8lD?{&3%-+U zRRgJ-m+w|Y6)S8QBxt#5gIV+UOiK-Uw6^0 zb04CE&hviBgJstJ>{C@h7E}Z%4QUUS$Zjfeg!GPzMP0;vF5o_emRjEpovqlJB#Ps5 zF>Li^s7zKnZ^`z>Ej@^Lioh?{$%$^Nm9krx-+EdlbZOE;!*uRI;G_a_w;D?wIs?o2 z*tjxIOY*zsKuU~Y^X2<$dA0iCnH2ErvryJ6l@1;O2!CFV*HbcBHck4A4-|#x)Aq(M zQZ@(55w8(mA$^ksV6B1Y;+w}~fB74K?65l4egJu)GaMY8IKlHxM4NC87pQ00a}K|2 z`a1SW54tObKOiSpYZBoa={M{hjPZunMsA=UE(YUUYsGBui z_ZP6O06E*JnsC`f9On`$yQ7lSY2~uD_43y}M;*~UxpH;>_GTp2su~4u*z`tt+&KoD zR3I|wKntl8dI|wd9DPta7+$Ex&M?`_>pGXLA8CGi*&)g@HQi31A{sPd65OV-JT^KN z=uUL4Nt?&cMC-L6MXr32Iklm4rub%@KLkPvCPha>DjsuskB%gyuD6VfEiU9n%vPhL zbohgl!wf}nbVP~I$gX3D#VXM7IoO2zzE^pv*=oYgA40pp;*VJi`yWa^ucX^yZ-8{X z@z9sCQ)(|!4YH2Pv^$pTU42HQuKlEQJ9rhQ_JC%M>y4}vGqb(7pJDu|#{BU%k#IT1 zpX=!dAR2G!*<;ely+b|jI9Su*Q|%q}?lZ9?!|`=DHFf>w>r*#Ul) zE6_-fHaG3%QM8=XZuOiexB#-aw-}lF^3GgSl_c%f4`WTQfv0)$bdZVOfhSW>AXRYEYTT8~Qybrja*G$qXiqI8GvP|iEvihI`y)xlj31wmVgY|qcvdDRCYmUpZ60mz z)WWNo>1m)4zO}Cab6D{%r;KA(8`)vra8M~F8q|e}x-+EN6t8e^J`=dAy``zYEIyQQgn%f-)460 zfSiP?=Ge7cQn~AwsRc9*5;N`uni~zyl%56UUXwmnTcv$RFWtp@YA1CjW*$TJXB)^V z0po^$7Y6?AOYHFP!2weC)bMu+})|g>A#^k zNI=j3tu7xK_(1WS(#LNOAJ8lYoQ%W%{Spvwh8;;%fi0`|`TJvK()D*DsD9|6YGLs;a|A&kUXL9q<<`2H_jac`$G;PEj-z&0;Q z620F52{8IcDyTxfk@>7IvVD*c+AQJvig80*EOOB}jrz6$S>(?t>X)i*O6*IiGvz)l zIxgqq%;+zQ_?#K6#Igis7zgPntpDus0t+pDtIs$7Pd_e#Gb9y~^|<<#Yjx=J8VV=ujH^~-rbsi1QKxHBmTrKD3ZJmh{D#;NOL~>PK8r8*#VtSPodYJKEgs6hkgUU<~ny=Zgr(tEY&~arRYQMh_J_F?C1IdX;A8F z;on@b(D0d7GWDn@_czXQ-4zBWi7T`D!ZGppq6nfyGdytmVXQ?1*1X*AOmLw1`T?{& z9Epvtjta28H{;6R`(dWq(#So-Y?CBAw_XATaU3_0eiYp4Y?Dj zbi(zqP_8v4L3E*R!T84;by_vbxv?n&Zp_)tLlf)36rpA2_k7#C45d-m-HRoz8`9rw z>U0>^Ss#mVUVIY&;QsYV>s13d&#{8Wx z=4oOlu$&F2;~ISO$4#bn9T+&{9-h+N$T|kpRueKEp=Z7h%CY)VA2~hKgrVb4J-BZn zZ>X#-0(3I{NO>q$r)kyU-4a-{Cf#!N>+x(y=D$3%|8ac-1JF6QU)z1}!sck!JP%7T}?a=Qv)YbK-q^M|QG{ zp6$%mE*2G^#T>78Hn1=7+EAddeQgFRwThG1>z7aQx=hy~_vF=>%NeqmI6q zin{|Hi^&5h_3IPjC|bx~t;C;?WX^5SIj^_?z;F8yS%9$V6IjoW1oJw(FvhXRc9AeJ3#(O{eB0m$Qj4@cZwIOsA_OkgkSY45OI)7MHZV ztX8zJN>N8N`;~-HtPmX|vmS19eT;r2$-(YTq}t7_i!RL>_LM8&RNs*~ms^;R_$m>{ zWT#E&?}Jv;cQJw7t4DcNonp9G!ml{o(bksN4pdht2TpiD|8hJ&5R|dZ0)#jBFe);Z zHAxZ=ZRLvN&R^!3Y7Q5tQSK0_qIsI`TTQ(F2FTu@9!Jc+fX!QE9-9z#@X+x3CtvI> z1cz;goO_a{JEeY1h{g?Sf7C^IQuV6pEBJBMi z?IHoGNr=7h=Cfn@YO3Lbw&so8oV&fjksdQVKLyb2>ip*DjVU~v4pv9@Si$=FnL)Xe zgroyWZm*Wx?|Z!)zrjqua-{ISSN8bgi8GChk2HOnV-{M8 zjIZ3Lao}Zq_r|3sdd>B^o`(=yL3DRh;RaoD29aVt6YSB%)HQO7!>PE4lS1yk3zB9g9&M)??{5Ufe34LN;KX zfz4Nc-v!_78?UcaZ)e9c!A!B12jk>yiyVJ{^O8pX)DC{DhWjQ%)_v8fHj}I z)$7OCh+>#}3a=u^KvDa18C!DIaicfbShBXJjrskbp{xO5^aD=`-MO)L616e?GiCZ9 z924@6%?9glAuePK)a!bj9r`p_C{^)od+VST5+HsjFwhq{uOBbR%(BMCLI~e)6aP?U zHvU6<{^8FvYN2?Q(HZc*#|Yj4pPf8+3{q|F@QdwjtFkou>MM1&i4VT-TBV0>g6-mC zz(f3J(tPqSN`h30SSWOW?NdhsY3e?hqtX8?EmtU)kM_66P7KM;BKGE>+xsLkB341;}T z0CT#4C+IUO@6pRih?x`JmKEd(^JVDaLgqn7PFFnnl{jhVBIOP$iu7P7=PyWx&R-CC zlnQu%|NVIt)M>)wCxB7s%vdquU^?ElBT*C``=&e6+HzTHNSvDzDR(5IT~KsjAl?56 zh!kW%E3}BInH&gA7f2eF$o@4V{%>ayLJ=8d1ZJHjkXJqbS&hO={&O;`F7)K@HR9?` z*F1~gL<>~IkE&E<HU&r~$b5n7R zH62)#9&+R+1-D3)U5d~F*G?8QgcRdhw=NA?t2RmIqkYExm~q?5 zfp?WQAA3#Cg#0S@j?JeODNs;V=&LD{;4dpw%LNSt>RtZST5VWE_$ODfp2g+}Z1}ZQ zZ`H5>onzZot{q>G#&-kb9iyodvhp$o9g*p`zRTx%U+udWKWe{8ws0*QmxfVa=Ug}4{1a_IGjq%EX4&f8l-iscPh0JgIM>NN{#x6WJ6wC@WK~W7!S0cDq_WjH& zs^z72Sf+!V{1t{UbA2ZDRcT7eLQUFo=Y>0>3iPElRMx6% znScl3D50m-VtB(eGyHUHNlVCun(*~r`_5T`FVvdu83-lFk$2RNejJo%GmyK_%2qIK zJoSqw%IEFiEA2#^*op5iUn*IT*5{LX$Ct^k2?*RX1Yru|cd#kYI3-^Psr8i^WzrT=fs}Ml$9Dj-nA7`#hV4* zL;UH1ZYQb>T?%aRGhc)G#vAxMaOo#;&+2qEm4}NK?EO5CDm@#O9k@cda_Pjh77vTf z6mUlIJEb0}eoGqv^yl`4_bP2?WA&#R$6wxOg!n)4b=`dP1?GsLn@!fQOr3k-#l5wq zQku+v^OC_2!JyjZe*RtR8U?hILrlR1-0NpDn%L@v1m)RRaPl+MN8EI&0~X3`?6}XD z)_Zh1?dp1HWw;yQ<-;%-z*N7H65ud5!|KSzS$4&%MVf)xUG&5Q^q+IWNy#-kfF6@OwamdT7Cl z58V%~!f{25^-Ib=E}Swu|M6Xt$C(_N@F%dhTZhJM0p!XyNC^&p9>& zU`>A}GnoO@{(L}O@FG$H$J`nq(F$X9g~LSIO>gB#22N4SzfY58YShOw=Cb&%kJmqU z9UpyoJwin|mc{ye!y$u_iVbZqlPa)MF@t?V2=3NRNn>Xs!$`$oMD5U!x+XAxO8Zl% z(c?*>CE7K{fQLA<`JU<)7)#IRsg=zT*i-S*D&s@oUWe46S?!|DEVcUnl-xjESXe`_n8vi=z{A^Ga*3@Zjxn1S- z{t6rusgFNBc0F!hvCTarWl!Yt_hv{S(4muX+D88N(M1ggGCL%!84jB1NtX2%3OAqi z?n?KM+Jjwy9xj0#Q|tol@tHu&bBB^fT(0f#7@8R8`3n>?tG;T#6`ziW`4T+|Lv}dKgYSbwI!fQZ zxO0u0$|rYeW5wPIOx)M8b-C8hzu%XNXf^^J@lCyIGEF8~PF-*t$91n^fXwRa(r!?d zG`gd}T`$8Rvz7SX##=!oYP*Q5F+-#b?;5td>7&a=uRBI;G2vpU5oHN{%}OvF`G-7! z!)*HGS&YL?f1qlrl9ZLl&foaMy3<+8VNAJ3yWq9DCb4Czc?VISjmnXwIJBaYnbhaM zfv+#uZe){v^y}As2*gWtA33=F=7ziy*z0F7o@oK1E34k$_4%wXuCs+dj=xD;f!CEwmhIKzy_{-?Ms#neOCMMU%ec)yXh1S`DL zJ$!YDoDfU5v(%gYXLS}$`X3g<9vvF~oV*dU+XL}?!m(2_vuVHTb+r-~)llin&Wnhrl>1~UbKB*)DMuX&qMy)GS;j=gcSQeN zqqIS7^sXH15w!UpRHjav&**s3yuWG_nRN9VP4>9^n&QeGe{);k{HNu(_BV2jb`O~M zmAPZeXfcxN8#E>GZt+cL1#eb@+W8N&pVUL+j}K0a1T3!OwB}Q)TS}g7>(mUI-5Wk1 z(Z^dn{Zcz9Xy(R12pzw5!%NV=A{=T|g3o#CIDzp5^?K0EMn^SG&0VSkoZHV&pd)9175n7@P%DNFO~&c{Jm( z-)5$1$)mQt(`b)$uYKTyy>RC*h)Oc~kRudN)9P(Ohl?dQ6yzqEjTew6t(Ux?pZTgP zqs;?Um_Ll{f~mJo9_$JdEAX)73$Opgm)c3z*AA4Y3)c9So@5KX}<_yUpX-`)+N7KMUXR*o!_V2#A z+!D!&1QrdNY#OsXdWR)b@P2b2Z>cDg6s?tf49_x8WQjgB1w)cOc8R2WYfDnV)pq&+EdE31mo z>UiZsoQPcRfL24$^a8`hJrzOKqSiHkiMnXM8kgdwB>D(rFv1zL6{!Kvb|3G+FF73R z5H+eR!YMuSe_5-C>o{xp{A@E`PIA;IIAlgZ|6p_hJyxF#(9&O*oaW;H=Qo9LLr@mP z^0#+Di6fKb2RY7&B3P4o?Xb+mJ%BLdz>o*2MU?Gp5jtE4Jj+>t3|j=LEq_LF14#D+ zA@nbZswbLA1vnN1;M}j{X~=%72m&3T?g5(S3Njozmlr4ugn`Ya0e200w2NrIN7Mk- zc`b~53bD6Bx#dS(Cy+c&&$L0$_BhFU+;*vY6B~Nz+*#Jd6%C;LxVxfW;{dOAD+#^8f631daCOXc*j*0#0J9Ch`D~*H>-tzaRv7 zc*!tf#`JaN3VbCBS-ihW9tv*mYbOAd}$tMKI|{K z;PjiloBe6$lC0CDIKLfwXx%hWijXqLOuHswg_U|a-GxukS>*7s%+FTV0LyCv0UXQY z`faXnj;eB8t*xR_+QdELWs-A0dz2W zM;k5%=s_#!UEcZgq_jYx_RET)j{_vQD(&#<7KL z3=oJQ0rZr@URjKOop_&re11Vq_<6hrNwCOdM6Wo2S`N()xN~C!*A`ys=v|Ehd!GbH zCC@LPC39lp&Xgr41VpT8tN^(bZZD)BDGhLtstIP~+8*#JHb@Hqhe7RHMz>S9pC*mZ zJ+~4y9PF~z?7Lj|Ack^9&pVUKllisbv{=USPo3>p6oS>s0IMw zK>CC;nIHLoIg=7w`f+MP1U)Y-pHrsL3?tMiRgkgn>Z=F*)NWs=t_9s!V(nF!1;Q|a zeXhl0C8!3emL(fH#+I^(sj{oZ_be?BI?qz*@8^L(q7kl-+|W*rh(cK-2mAPa+12si zg3>g9*pxW2QnPci&WZ@R#n=68Q+igvOyE0ZP5na8+?@x}*nuy_0Is6Ef?y*YsVKjCzfxxk(wPH?aCFlbVea zm(G^GeVh?>+lF!L9GaJ*v^p>b{|l+CyI716FB7HXC%_X=C-!&jcJ{MwKr z0vsj{N}Hy&@+cp6jUdzvr4Ylrq^7AAyH>JaJyxxp=y;Z@KSw14d9^~fFzb@v@Rk9FDy)4lAt zKV&k9J8dhJ6VT_|hv(HefoPTth(D_mgKOcT4*4ebRfb3B&D!%sCHAdcLxtYOgaG~G z^^C(F*vW`3XjJVQfl_OD%ae4grK)t=Ofk;Dcp;!zDduZpB5?5_N6puZI07dXdkRi=vc%PG{IUa&_a`j zZvQ|Nru+eT`H8PFb(&Ti&LZ`#LOHf^;dtq#uhj*0(fJtG!62w&tY7s8fHTF0ni0C0 zqzkDHx1QkIH+E~kV0(~#3?X)m0`+AT_CEZt&(U61X3=i<00I5y6y==iw+4R?+!4W% zx56#9ZSHq&xQyFfnkTdAr!@VUu6NK>|2v_kSO)Yp^g=A5oLnFOhvN_fEA&f={g!V9 zOslDUIFqyEjB|Mm9}lxyD2i_k;0~ugb`E1iKNb1>+~pKZpZUkfyZ9 zDe?UpcWI4+%h(!7p0>|WNhexWE_iX0Z$3GO!->p_{5?SqSMEDL1RunQv?uW7!c~WM zKOP41O=`X3>n>ZGdtZ??%aZR#CGFnEtm>{Vn0dlK{m(C$IC(;l91u{?)JTgKhEZJ0h1|_ zKu0RNA1J>bIAF$13`$v$+lRIzf_SUr;Ev>TxTQjC_9@LGuhYj$R5nkZTztqU9NZyn z8JDjBQ>`MsB|gJklNv3ZBW!jDi?mDjcKuigy9T+hY-2YJDuwuQF0$VI9N?LcBUXix z?~ArGqlIX-qES5d(>#M-l&9OG62;QrsT606uwrlKh)DFR1Mmnq+!(d}1=&3d2>Dc? z!~kZ71L$Y>;T%|gfJJEl3NdD2WGRy(khdE*)Z}4oG2QNi@R1+C!XSL8JvPok2T2T4 z6b?A;XAa276L{kCymU15pA{YS?V=^27-)H3H8xWKwvG}QkzEWY9+tJjw_=h!&)pTV z_1z8yYsg7QLbujo1xNbCkPXn8oUG`eG(dl13q7m$<%L|V-JNLtD=PbyNk(tQ1F~}{ zp^p!Dk)={|#==o&MRV<0*fEf2L1J>tnHA#1#CR|PC;(4B+Q9T3?lad>Z$8Ipq>uWRv5ogG;h9erD$%_`@*Ym zEz%Lvgp`4zb%Zf_O>8<@3>R-niv^80GbTQdIcNL)`ST~rWmypFYZ*UgDHqU@nLCeE zjCv7RXRkiwc3K%xzCEI%9?|!Aot#S+A^#!^kHQ^_95Vz)D$w6&?jM-B<|c}2c}sTh zX#R}JV5Z@>j!Cm%CcRV9a>AOcEyG53-ws)(*uL9%q`gx$lyFwo@})YDc>FDOA@d`z zFS!erkNRoi$s-_V-`+zUZGxnqV~CTue<+!#Tnzrf&hl1i$xs$0h6Lza(kP;)f=0*z=OH0;cdB3H^ zHqdjuvp4yyJ-h&h0Fw9pt=BDv{vi?h(rgxJ6U@HM=kNbAIVnpqXiO&jB>_Ie24@;q zZNQFOG%8I^D95x7UM>rRpYl%F>{#IE=eGD%H(#fO`E1M zH@OpekMvQ2Ff|wG*DXzsNUCel-8aBLo0+`wvDp6QZyGh-N!`oZz?kzP4}?VbZD9l{d{ITgQ$NV)Fm5+FwjL#Zv1`eBX^zm==zjA?1{Y{t0kJB0K)t*2v zNj>^!{P|<{F<^#269jDbIGcXCen7wB*Ybv%aJ{{cWW7#hKJHw&z3u4CEAJ$do?i5q zNnrf(ZC&}J^>1#k`_X>9YT4$8XQO?@X0PHNY{77oH&|_BZ{HtX&<|TON7w@n!R857#`M`t0_L#>L!|FROf< z^WZW24d7nN7W4VQ4cn*Z%Kf`{J7~k5ZrO`Unt~nbs*Vahe8(E>gwEF9{H^l&L)u|= z`;Ql080(_#|Dlc(LX z-#u^Uvj%G`=$=%*%ng8trrCQ+C8}V-*6o2qP^b^=eHL8E{ ze7)zo>W4trC4ES5F4@n2`;KR+Y>%tW^MIu%la4$nkyK}%!}okG@Xij|>wl_!|2_N{ z`ZK{(Wm1uE-HH=yxe>~ z&WeFw|A$E7$L2$Ezgm*NuABdDqven5t3LTDY?uGZnSJ@6;J(*7^O22r4%eJ(^kP^Q zu*j^4UE#~|;5iKZy}P7-)qiXJ&oDXnN9uoum^J&q^juTj4@`ApJ33-s+sXGa9q{;g z+IZ8tN=2Up;l7)aUq#>D7qt`Q7pu{|WTU(E6&rW2sG?ZWoxeA0cKq3(qGe>X_rQa1 zz%fIn@XPx*1J5gX-+BM*d;5QrO@Vur4r~2*{I6-AG_b`A>~sD&TYdZ>u(P0ka6aGH zxy<2LSdU)YvPt*$t-mLl6@AD#;4!H!cDIAOL4SebOce0aXF7`OrJ()H-_ z-&z$1^_E5ZCLjJ3&7LTHQsu;(oews3%+NdrJR13{ZT7w&;=k=*{p0+2#7^m>-10=_ z$YYPLakG{4z3b_T*`>!2X2EQ!zTy}M`y~^k9Xp`&q(nd7f4i;lF=#{H4I5xWKem3O z_L2P=^P9`>3oXBF%F7?HZTpp6fdd!siABsZ@=g&AQ*5-Gq44dvf;`WY>sN&2e;oYs z{9m*nFj$}Md=F`2g~OU+>+AAc4CXf`KYagf&g_=z)@KL*?LPatc>C|!KaR(gpEqIr zz+0>OTW~UP4ywH4SjBIy`WFWy+rNc<-0`1b#`1Ri4@&j_jN>l Date: Fri, 30 Jan 2026 17:35:09 +0800 Subject: [PATCH 73/83] [Feature] Support DiT Layerwise (Blockwise) CPU Offloading (#858) Signed-off-by: yuanheng Signed-off-by: Hongsheng Liu Co-authored-by: Hongsheng Liu Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .buildkite/pipeline.yml | 1 + .../diffusion/cpu_offload_diffusion.md | 82 +++- .../image_to_image/image_edit.py | 13 + .../image_to_video/image_to_video.py | 13 + .../text_to_image/text_to_image.py | 13 + .../text_to_video/text_to_video.py | 13 + .../test_diffusion_layerwise_offload.py | 110 ++++++ vllm_omni/diffusion/data.py | 6 + .../qwen_image/qwen_image_transformer.py | 1 + .../models/wan2_2/wan2_2_transformer.py | 1 + vllm_omni/diffusion/offload.py | 371 ++++++++++++++++-- .../worker/diffusion_model_runner.py | 17 +- vllm_omni/entrypoints/async_omni.py | 2 + vllm_omni/entrypoints/cli/serve.py | 11 + 14 files changed, 613 insertions(+), 41 deletions(-) create mode 100644 tests/e2e/offline_inference/test_diffusion_layerwise_offload.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6d731f127..49d1b2ece 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -77,6 +77,7 @@ steps: depends_on: image-build commands: - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py + - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py agents: queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU plugins: diff --git a/docs/user_guide/diffusion/cpu_offload_diffusion.md b/docs/user_guide/diffusion/cpu_offload_diffusion.md index 533b6b3b9..1f82fd608 100644 --- a/docs/user_guide/diffusion/cpu_offload_diffusion.md +++ b/docs/user_guide/diffusion/cpu_offload_diffusion.md @@ -1,19 +1,32 @@ # CPU Offloading for Diffusion Model ## Overview + +vLLM-Omni provides two offloading strategies to reduce GPU memory usage for diffusion models, allowing you to run larger models on GPUs with limited VRAM: + +1. **Model-level (Component) Offloading**: Swaps entire model components (DiT transformer, VAE, encoders) between GPU and CPU. +2. **Layerwise (Blockwise) Offloading**: Keeps only a single or a few transformer blocks on GPU at a time, with compute - memory copy overlap. + +Both approaches use pinned memory for faster CPU-GPU transfers. For now, the two offloading strategies could not be used at the same time. + + +## Model-level CPU Offloading + +### Implementation + CPU offload lets the diffusion worker move large model components between GPU and CPU memory on demand. It keeps the DiT transformer resident on GPU only while it is actively running, and swaps it out when encoders modules need the device. This reduces peak VRAM usage so bigger checkpoints run on smaller GPUs, or multiple requests can share the same GPU. -## Execution Model +**Execution Flow**: 1. Text encoders run on GPU while the DiT transformer is offloaded to CPU. 2. Before denoising, weights are prefetched back to GPU, honoring pinned-memory copies for speed. 3. After the diffusion step, the transformer returns to CPU and the process repeats as needed. Transfers use pinned host buffers, and the worker coordinates swaps via mutex-style hooks so components never compete for memory. -## Configuration +### Configuration You can enable CPU offload in two ways: -- **Python API**: set `enable_cpu_offload=True`. +1. **Python API**: set `enable_cpu_offload=True`. ```python from vllm_omni import Omni @@ -23,7 +36,66 @@ if __name__ == "__main__": m = Omni(model="Qwen/Qwen-Image",enable_cpu_offload=True) ``` -- **CLI**: pass `--enable-cpu-offload` to the diffusion service entrypoint. +2. **CLI**: pass `--enable-cpu-offload` to the diffusion service entrypoint. -## Known Limitations +### Limitations - Cold start latency increases for over one minute for some models(e.g., Qwen-Image) + + +## Layerwise (Blockwise) Offloading + +### Implementation +Layerwise offload operates at transformer block granularity, keeping a single transformer block, or a specified number of blocks, on GPU while others stay in CPU memory. + +Unlike full model-wise CPU offload which swaps entire components like DiT and encoders, layerwise offloading applies a sliding window way of loading and offloading weights between gpu and cpu: while block `i` computes, block `i+1` get prefetched asynchronously via pinned memory. In this way, only partial blocks(s) reside on GPU at any moment during inference, so that greatly decrease the memory occupancy. + +**Execution Flow**: + +1. During model initialization, all components are loaded to CPU first. Then components other than DiT model(s) in the pipeline, such as VAE and encoders, are moved to GPU. The weights of target transformer blocks are collected as contiguous tensors per layer on CPU with pinned memory; and non-block modules (embeddings, norms, etc) in the DiT model are moved to and stay on GPU. +2. The first block(s) are transferred to GPU during initialization of `LayerwiseOffloader`, before the first denoising step of the very first request. +3. As each block executes, the next block prefetches on a separate CUDA stream for compute - memory copy overlap. After execution, the current block is immediately freed from GPU memory. +4. When the last block completes, the first block prefetches for the next denoising step. + + +Example of hook executions of a DiT model with n layers, by default keep a single layer on GPU: +| Layer (block) idx | forward pre-hook | forward | forward post-hook | +|-------------------|--------------------------------|------------------|---------------------------| +| layer-0 | prefetch layer 1 (copy stream) | compute layer 0 | free layer-0 gpu weights | +| layer-1 | prefetch layer 2 (copy stream) | compute layer 1 | free layer-1 gpu weights | +| layer-2 | prefetch layer 3 (copy stream) | compute layer 2 | free layer-2 gpu weights | +| ... | ... | ... | ... | +| layer-(n-1) | **prefetch layer 0 (copy stream)** | compute layer (n-1) | free layer (n-1) gpu weights | + + +### Configuration + +1. **Python API**: set `enable_layerwise_offload=True` and optionally `layerwise_num_gpu_layers`. + +```python +from vllm_omni import Omni + +if __name__ == "__main__": + m = Omni( + model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", + enable_layerwise_offload=True, + ... + ) +``` + +2. **CLI**: pass `--enable-layerwise-offload` and `--layerwise-num-gpu-layers` to the diffusion service entrypoint. + +### Supported Models + +| Architecture | Models | Example HF Models | DiT Model Cls | Blocks Attr Name | +|--------------|--------|-------------------|----------|----------| +| `QwenImagePipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | "transformer_blocks" | +| `Wan22Pipeline` | Wan2.2 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | "blocks" | + +NOTE: Models must define `_layerwise_offload_blocks_attr` class attribute so that the layerwise offloader finds the target transformer blocks. + +### Limitations +- Cold start latency increases because of + 1) components are loaded to CPU first at the very first during initialization, + 2) weight consolidation and pinning +- Performance depends on CPU <-> GPU interconnection (e.g., PCIe bandwidth). +- Support single GPU only for now diff --git a/examples/offline_inference/image_to_image/image_edit.py b/examples/offline_inference/image_to_image/image_edit.py index 5ffab87f0..8f330e09d 100644 --- a/examples/offline_inference/image_to_image/image_edit.py +++ b/examples/offline_inference/image_to_image/image_edit.py @@ -295,6 +295,17 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Enable CPU offloading for diffusion models.", ) + parser.add_argument( + "--enable-layerwise-offload", + action="store_true", + help="Enable layerwise (blockwise) offloading on DiT modules.", + ) + parser.add_argument( + "--layerwise-num-gpu-layers", + type=int, + default=1, + help="Number of ready layers (blocks) to keep on GPU during generation.", + ) return parser.parse_args() @@ -350,6 +361,8 @@ def main(): # Initialize Omni with appropriate pipeline omni = Omni( model=args.model, + enable_layerwise_offload=args.enable_layerwise_offload, + layerwise_num_gpu_layers=args.layerwise_num_gpu_layers, vae_use_slicing=args.vae_use_slicing, vae_use_tiling=args.vae_use_tiling, cache_backend=args.cache_backend, diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 4fca7825c..178528784 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -74,6 +74,17 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Enable CPU offloading for diffusion models.", ) + parser.add_argument( + "--enable-layerwise-offload", + action="store_true", + help="Enable layerwise (blockwise) offloading on DiT modules.", + ) + parser.add_argument( + "--layerwise-num-gpu-layers", + type=int, + default=1, + help="Number of ready layers (blocks) to keep on GPU during generation.", + ) return parser.parse_args() @@ -112,6 +123,8 @@ def main(): omni = Omni( model=args.model, + enable_layerwise_offload=args.enable_layerwise_offload, + layerwise_num_gpu_layers=args.layerwise_num_gpu_layers, vae_use_slicing=args.vae_use_slicing, vae_use_tiling=args.vae_use_tiling, boundary_ratio=args.boundary_ratio, diff --git a/examples/offline_inference/text_to_image/text_to_image.py b/examples/offline_inference/text_to_image/text_to_image.py index 3e0628a15..a79e5d640 100644 --- a/examples/offline_inference/text_to_image/text_to_image.py +++ b/examples/offline_inference/text_to_image/text_to_image.py @@ -107,6 +107,17 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Enable CPU offloading for diffusion models.", ) + parser.add_argument( + "--enable-layerwise-offload", + action="store_true", + help="Enable layerwise (blockwise) offloading on DiT modules.", + ) + parser.add_argument( + "--layerwise-num-gpu-layers", + type=int, + default=1, + help="Number of ready layers (blocks) to keep on GPU during generation.", + ) parser.add_argument( "--tensor_parallel_size", type=int, @@ -172,6 +183,8 @@ def main(): omni = Omni( model=args.model, + enable_layerwise_offload=args.enable_layerwise_offload, + layerwise_num_gpu_layers=args.layerwise_num_gpu_layers, vae_use_slicing=args.vae_use_slicing, vae_use_tiling=args.vae_use_tiling, cache_backend=args.cache_backend, diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index b5c003dfb..63474987f 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -79,6 +79,17 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Enable CPU offloading for diffusion models.", ) + parser.add_argument( + "--enable-layerwise-offload", + action="store_true", + help="Enable layerwise (blockwise) offloading on DiT modules.", + ) + parser.add_argument( + "--layerwise-num-gpu-layers", + type=int, + default=1, + help="Number of ready layers (blocks) to keep on GPU during generation.", + ) parser.add_argument( "--ulysses_degree", type=int, @@ -128,6 +139,8 @@ def main(): omni = Omni( model=args.model, + enable_layerwise_offload=args.enable_layerwise_offload, + layerwise_num_gpu_layers=args.layerwise_num_gpu_layers, vae_use_slicing=args.vae_use_slicing, vae_use_tiling=args.vae_use_tiling, boundary_ratio=args.boundary_ratio, diff --git a/tests/e2e/offline_inference/test_diffusion_layerwise_offload.py b/tests/e2e/offline_inference/test_diffusion_layerwise_offload.py new file mode 100644 index 000000000..87a9e0a9e --- /dev/null +++ b/tests/e2e/offline_inference/test_diffusion_layerwise_offload.py @@ -0,0 +1,110 @@ +import sys +from pathlib import Path + +import pytest +import torch +from vllm.distributed.parallel_state import cleanup_dist_env_and_memory + +from tests.utils import GPUMemoryMonitor +from vllm_omni.inputs.data import OmniDiffusionSamplingParams +from vllm_omni.platforms import current_omni_platform + +# ruff: noqa: E402 +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from vllm_omni import Omni + +# Models to test and expected saved memory in MB, correspondingly +MODELS_SAVED_MEMORY_MB = {"riverclouds/qwen_image_random": 4500} + + +def run_inference( + model_name: str, + layerwise_offload: bool = False, + num_gpu_layers: int = 1, + num_inference_steps: int = 3, +) -> float: + # For now, only support on GPU, so apply torch.cuda operations here + # NPU / ROCm platforms are expected to be detected and skipped this test function + torch.cuda.empty_cache() + device_index = torch.cuda.current_device() + monitor = GPUMemoryMonitor(device_index=device_index, interval=0.02) + monitor.start() + + m = Omni( + model=model_name, + enable_layerwise_offload=layerwise_offload, + layerwise_num_gpu_layers=num_gpu_layers, + boundary_ratio=0.875, + flow_shift=5.0, + ) + + torch.cuda.reset_peak_memory_stats(device=device_index) + + # Refer to tests/e2e/offline_inference/test_t2v_model.py + # Use minimal settings for testing + height = 480 + width = 640 + num_frames = 5 + + m.generate( + "A cat sitting on a table", + OmniDiffusionSamplingParams( + height=height, + width=width, + generator=torch.Generator("cuda").manual_seed(42), + guidance_scale=1.0, + num_inference_steps=num_inference_steps, + num_frames=num_frames, + ), + ) + + peak = monitor.peak_used_mb + monitor.stop() + + return peak + + +@pytest.mark.skipif(current_omni_platform.is_npu() or current_omni_platform.is_rocm(), reason="Hardware not supported") +@pytest.mark.parametrize("model_name", MODELS_SAVED_MEMORY_MB.keys()) +def test_layerwise_offload_diffusion_model(model_name: str): + """Test that layerwise offloading reduces GPU memory usage. + + This test verifies that layerwise offloading significantly reduces peak + GPU memory usage compared to loading the entire model on GPU. The layerwise + offloader keeps only a single transformer block on GPU at a time, with + prefetching for compute-memory overlap. + """ + try: + # Run without layerwise offloading (baseline) + no_offload_peak_memory = run_inference(model_name, layerwise_offload=False) + cleanup_dist_env_and_memory() + + # Run with layerwise offloading (1 layer on device) + layerwise_offload_peak_memory = run_inference(model_name, layerwise_offload=True, num_gpu_layers=1) + cleanup_dist_env_and_memory() + + # Run with 2 layers on device + layerwise_offload_two_layers_peak = run_inference(model_name, layerwise_offload=True, num_gpu_layers=2) + except Exception: + pytest.fail("Inference failed") + + print(f"Layerwise offload peak memory (1 GPU layer): {layerwise_offload_peak_memory} MB") + print(f"Layerwise offload peak memory (2 GPU layers): {layerwise_offload_two_layers_peak} MB") + print(f"No offload peak memory: {no_offload_peak_memory} MB") + + # Verify that layerwise offloading significantly reduces memory usage + # Passes only if the actual savings exceeds the expected savings + assert layerwise_offload_peak_memory + MODELS_SAVED_MEMORY_MB[model_name] < no_offload_peak_memory, ( + f"Layerwise offload peak memory {layerwise_offload_peak_memory} MB " + f"should be significantly less than no offload peak memory {no_offload_peak_memory} MB" + ) + + # Verify that 2 GPU layers uses more memory than 1 GPU layer + # But not excessively more (should be a reasonable increase) + assert layerwise_offload_peak_memory < layerwise_offload_two_layers_peak, ( + f"1 GPU layer peak {layerwise_offload_peak_memory} MB should be < " + f"2 GPU layers peak {layerwise_offload_two_layers_peak} MB" + ) diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 286ab9153..4248071f0 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -288,6 +288,12 @@ class OmniDiffusionConfig: # - Text encoders run on GPU while DiT is on CPU # - DiT runs on GPU while encoders are on CPU enable_cpu_offload: bool = False + + # Layer-wise offloading (block-level offloading) parameters + enable_layerwise_offload: bool = False + # Number of transformer blocks ready for computation to keep on GPU + layerwise_num_gpu_layers: int = 1 + use_fsdp_inference: bool = False pin_cpu_memory: bool = True # Use pinned memory for faster transfers when offloading diff --git a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py index 054a5cb36..47098ff8a 100644 --- a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py +++ b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py @@ -758,6 +758,7 @@ class QwenImageTransformer2DModel(CachedTransformer): # -- typically a transformer layer # used for torch compile optimizations _repeated_blocks = ["QwenImageTransformerBlock"] + _layerwise_offload_blocks_attr = "transformer_blocks" packed_modules_mapping = { "to_qkv": ["to_q", "to_k", "to_v"], "add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"], diff --git a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py index e4ba6318b..ab92ad0c8 100644 --- a/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py +++ b/vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py @@ -531,6 +531,7 @@ class WanTransformer3DModel(nn.Module): """ _repeated_blocks = ["WanTransformerBlock"] + _layerwise_offload_blocks_attr = "blocks" packed_modules_mapping = { "to_qkv": ["to_q", "to_k", "to_v"], } diff --git a/vllm_omni/diffusion/offload.py b/vllm_omni/diffusion/offload.py index 174c64342..6f1d8a0db 100644 --- a/vllm_omni/diffusion/offload.py +++ b/vllm_omni/diffusion/offload.py @@ -12,7 +12,9 @@ This allows running large models on limited GPU memory. from __future__ import annotations -from typing import TYPE_CHECKING +from functools import partial +from itertools import chain +from typing import TYPE_CHECKING, Any import torch from torch import nn @@ -66,7 +68,7 @@ class SequentialOffloader: # Release allocator blocks when tensors leave the GPU. if previous_device.type != "cpu": - current_omni_platform.empty_cache() + torch.cuda.empty_cache() if self.pin_memory: for p in module.parameters(): @@ -125,6 +127,267 @@ class SequentialOffloader: self._handles = [] +class LayerwiseOffloader: + """Layer-wise CPU offloading for transformer blocks. + + Keeps only a sliding window of layers (blocks), by default a single layer, on GPU, + prefetching the next block while the current block computes to approach compute - memcpy overlap. + Unused blocks are freed on GPU. + + Based on implementations from: + https://github.com/sgl-project/sglang/blob/v0.5.8/python/sglang/multimodal_gen/runtime/utils/layerwise_offload.py + """ + + def __init__( + self, + blocks: list[nn.Module], + device: torch.device, + pin_memory: bool = True, + num_gpu_layers: int = 1, + ): + assert all(isinstance(m, nn.Module) for m in blocks), "All transformer blocks must be torch.nn.Module" + assert current_omni_platform.is_cuda(), "Layerwise offloading is only supported on cuda devices for now" + + self.blocks = blocks + self.device = device + self.pin_memory = pin_memory + self.num_gpu_layers = num_gpu_layers + self.num_blocks = len(self.blocks) + if self.num_blocks == 0: + raise ValueError("LayerwiseOffloader requires at least one block, but found 0.") + if not (1 <= self.num_gpu_layers <= self.num_blocks): + raise ValueError(f"Invalid num_gpu_layers {self.num_gpu_layers} with {self.num_blocks} blocks") + + self._pre_hook_handles: list = [] + self._post_hook_handles: list = [] + + self._copy_stream = torch.cuda.Stream() + + # Per-layer synchronization primitive: set after H2D copy completes. + self._prefetch_done: list[torch.cuda.Event | None] = [None] * self.num_blocks + + # Simple state to avoid redundant work. + self._resident: list[bool] = [False] * self.num_blocks + + # Pre-allocate gpu tensors + # layer-id -> {dtype -> flattened aggregated cpu tensor} + self.layer_cpu_weights: list[dict[torch.dtype, torch.Tensor]] = [] + self.layer_metadata: list[dict[torch.dtype, list[dict[str, Any]]]] = [] + + self.block_parameters: dict[int, dict[str, nn.Parameter]] = {} + self.block_buffers: dict[int, dict[str, torch.Tensor]] = {} + for layer_idx, block in enumerate(self.blocks): + self.block_parameters[layer_idx] = dict(block.named_parameters()) + self.block_buffers[layer_idx] = dict(block.named_buffers()) + + dtype_cpu_flattened_weights, dtype_metadata = self._to_cpu( + self.block_parameters[layer_idx], self.block_buffers[layer_idx] + ) + self.layer_cpu_weights.append(dtype_cpu_flattened_weights) + self.layer_metadata.append(dtype_metadata) + + if self.num_blocks != len(self.layer_cpu_weights): + logger.error( + f"Inconsistent block layers happened: # of blocks: {self.num_blocks}; " + f"# of layer cpu weights: {len(self.layer_cpu_weights)}" + ) + + # Register pre and post forward hooks on each of the blocks + self.register_block_hooks() + + # Pre-fetch the first layer + # For subsequent requests, the first layer/block will be pre-fetched + # during the last layer compute of the previous request. + self.prefetch_layer(0, non_blocking=False) + + def _to_cpu( + self, params: dict[str, nn.Parameter], bufs: dict[str, torch.Tensor] + ) -> tuple[dict[torch.dtype, torch.Tensor], dict[torch.dtype, list[dict[str, Any]]]]: + """Move block parameters and buffers to CPU, flattening by dtype. + + Consolidates parameters and buffers into contiguous CPU tensors grouped by dtype + for GPU transfers. Replaces original tensors with empty placeholders. + + Returns: + Tuple of + flattened CPU tensors by dtype, + metadata for reconstruction by dtype + """ + dtype_grouped_weights: dict[torch.dtype, dict[str, torch.Tensor]] = {} + dtype_cpu_flattened_weights: dict[torch.dtype, torch.Tensor] = {} + # order does matter + dtype_metadata: dict[torch.dtype, list[dict[str, Any]]] = {} + + for name, param_or_buf in chain(params.items(), bufs.items()): + dtype = param_or_buf.dtype + if dtype not in dtype_grouped_weights: + dtype_grouped_weights[dtype] = {} + dtype_grouped_weights[dtype][name] = param_or_buf + + for dtype, name2weights in dtype_grouped_weights.items(): + # total # of parameters + buffers + total_numel = sum(t.numel() for _, t in name2weights.items()) + cpu_tensor = torch.empty(total_numel, dtype=dtype, device="cpu", pin_memory=self.pin_memory) + + current_offset = 0 + for name, param_or_buf in name2weights.items(): + numel = param_or_buf.numel() + cpu_tensor[current_offset : current_offset + numel].copy_(param_or_buf.flatten()) + if dtype not in dtype_metadata: + dtype_metadata[dtype] = [] + dtype_metadata[dtype].append( + { + "name": name, + "offset": current_offset, + "numel": numel, + "shape": param_or_buf.shape, + } + ) + + param_or_buf.data = torch.empty((), device=self.device, dtype=dtype) + current_offset += numel + + dtype_cpu_flattened_weights[dtype] = cpu_tensor + + return dtype_cpu_flattened_weights, dtype_metadata + + def register_block_hooks(self) -> None: + """Register forward hooks on blocks for prefetching and offloading.""" + + def _pre_hook(module: nn.Module, args: tuple, *, layer_idx: int) -> None: + # For the last block / layer, prefetch layer 0 (the first layer) + next_id = (layer_idx + 1) % self.num_blocks + self.prefetch_layer(next_id, non_blocking=True) + + def _post_hook(module: nn.Module, args: tuple, output: tuple, *, layer_idx: int) -> None: + self.offload_layer(layer_idx) + self._resident[layer_idx] = False + self._prefetch_done[layer_idx] = None + + for i, layer in enumerate(self.blocks): + pre_hook_fn = partial(_pre_hook, layer_idx=i) + handle = layer.register_forward_pre_hook(pre_hook_fn) + self._pre_hook_handles.append(handle) + + post_hook_fn = partial(_post_hook, layer_idx=i) + handle = layer.register_forward_hook(post_hook_fn) + self._post_hook_handles.append(handle) + + @torch.compiler.disable + def prefetch_layer(self, layer_idx: int, non_blocking: bool = True) -> None: + """Copy layer weights from CPU -> GPU. + + Pre-fetch target layer in an asynchronous way with compute - memory copy overlap, + with non_blocking set to True. + """ + if layer_idx >= self.num_blocks or layer_idx < 0: + logger.warning(f"Invalid layer id specified: {layer_idx}") + return + + self._copy_stream.wait_stream(torch.cuda.current_stream()) + + layers_to_fetch = [(layer_idx + i) % self.num_blocks for i in range(self.num_gpu_layers)] + + for idx in layers_to_fetch: + if self._resident[idx]: + continue + + layer_params = self.block_parameters[idx] + layer_bufs = self.block_buffers[idx] + + evt = torch.cuda.Event() + gpu_weights: dict[torch.dtype, torch.Tensor] = {} + + with torch.cuda.stream(self._copy_stream): + for dtype, cpu_weight in self.layer_cpu_weights[idx].items(): + gpu_weight = torch.empty(cpu_weight.shape, dtype=dtype, device=self.device) + gpu_weight.copy_(cpu_weight, non_blocking=non_blocking) + gpu_weights[dtype] = gpu_weight + + evt.record(self._copy_stream) + + for dtype in self.layer_metadata[idx]: + ordered_metadata: list[dict[str, Any]] = self.layer_metadata[idx][dtype] + + gpu_weight = gpu_weights[dtype] + + for metadata in ordered_metadata: + target_name = metadata["name"] + target_param_or_buf = ( + layer_params[target_name] if target_name in layer_params else layer_bufs[target_name] + ) + + target_param_or_buf.data = gpu_weight[ + metadata["offset"] : metadata["offset"] + metadata["numel"] + ].view(metadata["shape"]) + + self._prefetch_done[idx] = evt + self._resident[idx] = True + + @torch.compiler.disable + def offload_layer(self, layer_idx: int) -> None: + """Free GPU memory for layer by replacing tensors with empty placeholders.""" + if layer_idx >= self.num_blocks or layer_idx < 0: + logger.warning(f"Invalid layer id specified: {layer_idx}") + return + if not self._resident[layer_idx]: + logger.warning(f"{layer_idx} is not residing on GPU") + return + + evt = self._prefetch_done[layer_idx] + if evt is not None: + torch.cuda.current_stream().wait_event(evt) + + # free GPU residency + for _, param in self.block_parameters[layer_idx].items(): + param.data = torch.empty((), device=self.device, dtype=param.dtype) + for _, buf in self.block_buffers[layer_idx].items(): + buf.data = torch.empty((), device=self.device, dtype=buf.dtype) + + def remove_all_hooks(self) -> None: + """Remove all hooks.""" + for h in self._pre_hook_handles: + h.remove() + for h in self._post_hook_handles: + h.remove() + self._pre_hook_handles.clear() + self._post_hook_handles.clear() + + @staticmethod + def get_blocks_attr_name(model: nn.Module) -> str | None: + """Retrieve blocks attribute name from provided DiT model""" + return getattr(model.__class__, "_layerwise_offload_blocks_attr", None) + + @staticmethod + def get_blocks_from_dit(model: nn.Module) -> list[nn.Module]: + """ + Retrieve a list of blocks from provided DiT model. Blocks attribute name + are found by `_layerwise_offload_blocks_attr` set to DiT models. For example, + + ``` + class WanTransformer3DModel(nn.Module): + _layerwise_offload_blocks_attr = "blocks" + ``` + """ + blocks_attr_name = LayerwiseOffloader.get_blocks_attr_name(model) + if blocks_attr_name is None: + logger.warning( + f"No _layerwise_offload_blocks_attr defined for {model.__class__.__name__}, " + "skipping layerwise offloading" + ) + return [] + + _blocks = getattr(model, blocks_attr_name, None) + if _blocks is None: + logger.warning( + f"Blocks (layers) '{blocks_attr_name}' not found on {model.__class__.__name__}, " + "skipping layerwise offloading" + ) + return [] + + return list(_blocks) + + def apply_offload_hooks( model: nn.Module, od_config: OmniDiffusionConfig, @@ -142,7 +405,24 @@ def apply_offload_hooks( model: Diffusion pipeline model od_config: OmniDiffusionConfig with offload settings """ - if not getattr(od_config, "enable_cpu_offload", False): + enable_cpu_offload = getattr(od_config, "enable_cpu_offload", False) + enable_layerwise_offload = getattr(od_config, "enable_layerwise_offload", False) + pin_cpu_memory = getattr(od_config, "pin_cpu_memory", True) + + if not enable_cpu_offload and not enable_layerwise_offload: + return + if enable_cpu_offload and enable_layerwise_offload: + # NOTE: Model-wise and layerwise cpu offloading are not supported together at this moment, + # consider layerwise offloading has higher priority than model-wise offloading + enable_cpu_offload = False + logger.info( + "Model-wise and layer-wise CPU offloading are not supported together at this moment. " + "Automatically disabled model-wise offloading." + ) + # For now, model-wise and layer-wise (block-wise) offloading + # are functioning as expected when cuda device is available + if not current_omni_platform.is_cuda() or current_omni_platform.get_device_count() < 1: + logger.info("CPU Offloading requires cuda devices available. Skipping for now...") return # Find DiT/transformer modules @@ -167,7 +447,6 @@ def apply_offload_hooks( if not dit_modules: logger.warning("enable_cpu_offload enabled but no transformer/dit/unet found") return - if device is None: try: device = next(dit_modules[0].parameters()).device @@ -175,7 +454,8 @@ def apply_offload_hooks( try: device = current_omni_platform.get_torch_device() except (NotImplementedError, AttributeError): - device = torch.device("cpu") + logger.error("Fail to get device of pipeline. Skipping applying offloading hooks") + return # Collect all encoders encoders: list[nn.Module] = [] @@ -184,29 +464,72 @@ def apply_offload_hooks( if hasattr(model, attr) and getattr(model, attr) is not None: encoders.append(getattr(model, attr)) encoder_names.append(attr) - - if not encoders: + if not encoders and enable_cpu_offload: logger.warning("enable_cpu_offload enabled but no encoders found") return + for enc in encoders: + enc.to(device) - # Initial state: keep DiT modules on CPU (encoders typically run first) - pin = getattr(od_config, "pin_cpu_memory", True) - for dit_mod in dit_modules: - dit_mod.to("cpu") + # Collect VAE + for name in ["vae"]: + module = getattr(model, name, None) + if module is None: + continue + try: + module.to(device, non_blocking=True) + except Exception as exc: + logger.debug("Failed to move %s to GPU: %s", name, exc) - current_omni_platform.empty_cache() - - if pin: + if enable_cpu_offload: + # Initial state: keep DiT modules on CPU (encoders typically run first) for dit_mod in dit_modules: - for p in dit_mod.parameters(): - if p.data.device.type == "cpu" and not p.data.is_pinned(): - p.data = p.data.pin_memory() + dit_mod.to("cpu") - # Register sequential offload hooks - SequentialOffloader(dit_modules, encoders, device, pin).register() + torch.cuda.empty_cache() - logger.info( - "CPU offload enabled: %s <-> %s (mutual exclusion)", - ", ".join(dit_names), - ", ".join(encoder_names), - ) + if pin_cpu_memory: + for dit_mod in dit_modules: + for p in dit_mod.parameters(): + if p.data.device.type == "cpu" and not p.data.is_pinned(): + p.data = p.data.pin_memory() + + # Register sequential offload hooks + SequentialOffloader(dit_modules, encoders, device, pin_cpu_memory).register() + logger.info( + "CPU offload enabled: %s <-> %s (mutual exclusion)", + ", ".join(dit_names), + ", ".join(encoder_names), + ) + elif enable_layerwise_offload: + logger.info(f"Applying offloading hooks on {dit_names}") + + for i, dit_module in enumerate(dit_modules): + logger.info(f"Applying hook on {dit_names[i]} ({dit_module.__class__.__name__})") + blocks_attr_name = LayerwiseOffloader.get_blocks_attr_name(dit_module) + blocks = LayerwiseOffloader.get_blocks_from_dit(dit_module) + + if not blocks_attr_name or not blocks: + logger.warning( + "Target layers (blocks) are not found. " + f"Skipping offloading on {dit_names[i]} ({dit_module.__class__.__name__})" + ) + continue + + # move modules other than blocks to gpu and keep them on gpu + for name, m in dit_module.named_children(): + # Skip the blocks module (layers to be offloaded) + if name == blocks_attr_name: + logger.debug(f"Skipped module {name}") + continue + + m.to(device) + logger.debug(f"Moved {name} to device {device}") + + # set to the module (transformer) + offloader = LayerwiseOffloader(blocks, device, pin_cpu_memory, od_config.layerwise_num_gpu_layers) + setattr(dit_module, "_layerwise_offloader", offloader) + + logger.info( + f"Layerwise offloading enabled on {len(blocks)} layers (blocks), " + f"with {od_config.layerwise_num_gpu_layers} kept on device" + ) diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index b60d2c2f5..b0bec0852 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -76,7 +76,9 @@ class DiffusionModelRunner: memory_pool_context_fn: Optional function that returns a context manager for memory pool allocation (used for sleep mode). """ - load_device = "cpu" if self.od_config.enable_cpu_offload else str(self.device) + load_device = ( + "cpu" if self.od_config.enable_cpu_offload or self.od_config.enable_layerwise_offload else str(self.device) + ) def get_memory_context(): if memory_pool_context_fn is not None: @@ -104,17 +106,8 @@ class DiffusionModelRunner: ) logger.info("Model runner: Model loaded successfully.") - # Apply CPU offloading (DiT <-> encoders mutual exclusion) - if self.od_config.enable_cpu_offload: - for name in ["vae"]: - module = getattr(self.pipeline, name, None) - if module is None: - continue - try: - module.to(self.device, non_blocking=True) - except Exception as exc: - logger.debug("Failed to move %s to GPU: %s", name, exc) - + # Apply CPU offloading + if self.od_config.enable_cpu_offload or self.od_config.enable_layerwise_offload: apply_offload_hooks(self.pipeline, self.od_config, device=self.device) # Apply torch.compile if not in eager mode diff --git a/vllm_omni/entrypoints/async_omni.py b/vllm_omni/entrypoints/async_omni.py index ec2a18ac4..ab829beaa 100644 --- a/vllm_omni/entrypoints/async_omni.py +++ b/vllm_omni/entrypoints/async_omni.py @@ -165,6 +165,8 @@ class AsyncOmni(OmniBase): "cache_config": cache_config, "enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False), "enable_cpu_offload": kwargs.get("enable_cpu_offload", False), + "enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False), + "layerwise_num_gpu_layers": kwargs.get("layerwise_num_gpu_layers", False), "enforce_eager": kwargs.get("enforce_eager", False), }, "final_output": True, diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index bbd167111..5613bdaeb 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -199,6 +199,17 @@ class OmniServeCommand(CLISubcommand): action="store_true", help="Enable CPU offloading for diffusion models.", ) + serve_parser.add_argument( + "--enable-layerwise-offload", + action="store_true", + help="Enable layerwise (blockwise) offloading on DiT modules.", + ) + serve_parser.add_argument( + "--layerwise-num-gpu-layers", + type=int, + default=1, + help="Number of layers (blocks) to keep on GPU during generation.", + ) # Video model parameters (e.g., Wan2.2) - engine-level omni_config_group.add_argument( From e6b9d529053a81c7d737e56a2b548fb439bd4b7f Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Fri, 30 Jan 2026 20:50:23 +0800 Subject: [PATCH 74/83] [BugFix] Modify max_tokens and modify the log and fix #1103 (#1097) Signed-off-by: amy-why-3459 Co-authored-by: Hongsheng Liu --- .../distributed/omni_connectors/adapter.py | 8 ++++---- .../connectors/shm_connector.py | 18 ++++++++++++++++-- .../qwen3_omni_moe_async_chunk.yaml | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/vllm_omni/distributed/omni_connectors/adapter.py b/vllm_omni/distributed/omni_connectors/adapter.py index b03202525..977ecc88b 100644 --- a/vllm_omni/distributed/omni_connectors/adapter.py +++ b/vllm_omni/distributed/omni_connectors/adapter.py @@ -245,7 +245,7 @@ def get_through_connector(connector, target_stage_id, stage_id, req_id, connecto if payload_data: connector.request_prompt_token_ids[req_id] = payload_data.get("thinker_input_ids", []) connector.get_requests[req_id] += 1 - logger.debug(f"[Stage-{stage_id}] Received one chunk for request {connector_get_key}") + logger.debug("[Stage-%d] Received one chunk for request %s", stage_id, connector_get_key) break time.sleep(0.01) return payload_data @@ -325,7 +325,7 @@ def put_chunk( logger.error(f"Failed to use custom_process_input_func for payload extraction: {e}") if not payload_data: - logger.warning(f"[Stage-{stage_id}] No payload data to send for request {request_id}") + logger.warning("[Stage-%d] No payload data to send for request %s", stage_id, request_id) return if stage_id == 0 and chunk_id == 0: @@ -341,7 +341,7 @@ def put_chunk( payload_data["thinker_hidden_states"] = torch.cat( (save_payload.get("thinker_hidden_states"), payload_data.get("thinker_hidden_states")), dim=0 ) - logger.info(f"[Stage-{stage_id}] Merged embeddings and hidden states for request {request_id}") + logger.debug("[Stage-%d] Merged embeddings and hidden states for request %s", stage_id, request_id) if stage_id == 1: # TODO: Make parameters configurable and optimize algorithms @@ -367,7 +367,7 @@ def put_chunk( if success: connector.put_requests[request_id] += 1 - logger.info(f"[Stage-{stage_id}] Sent {connector_put_key}") + logger.debug("[Stage-%d] Sent %s", stage_id, connector_put_key) def compute_talker_prompt_ids_length(prompt_ids: list[int]) -> int: diff --git a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py index bd6ec8551..9ad32e1c7 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import fcntl +import os import time from collections import defaultdict from typing import Any @@ -53,7 +55,12 @@ class SharedMemoryConnector(OmniConnectorBase): # if size > self.threshold: if True: # TODO: correct put & get logic # Use Shared Memory - meta = shm_write_bytes(payload, name=put_key) + lock_file = f"/dev/shm/shm_{put_key}_lockfile.lock" + with open(lock_file, "w") as lockf: + fcntl.flock(lockf, fcntl.LOCK_EX) + meta = shm_write_bytes(payload, name=put_key) + fcntl.flock(lockf, fcntl.LOCK_UN) + # meta contains {'name': ..., 'size': ...} metadata[put_key] = {"shm": meta, "size": size} self._metrics["shm_writes"] += 1 @@ -97,7 +104,14 @@ class SharedMemoryConnector(OmniConnectorBase): return None, 0 try: - data_bytes = shm_read_bytes({"name": get_key, "size": shm.size}) + lock_file = f"/dev/shm/shm_{get_key}_lockfile.lock" + with open(lock_file) as lockf: + fcntl.flock(lockf, fcntl.LOCK_SH) + data_bytes = shm_read_bytes({"name": get_key, "size": shm.size}) + fcntl.flock(lockf, fcntl.LOCK_UN) + # Clean up the temporary file if it still exists. + if os.path.exists(lock_file): + os.remove(lock_file) obj = self.deserialize_obj(data_bytes) return obj, shm.size finally: diff --git a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml index 1a8b60aa1..c8cb67dd5 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml @@ -63,7 +63,7 @@ stage_args: default_sampling_params: temperature: 0.9 top_k: 50 - max_tokens: 4096 + max_tokens: 2048 # TODO: The max_tokens of the async_chunk feature cannot exceed 2048. seed: 42 detokenize: False repetition_penalty: 1.05 From be835c448353675bba2f73ab1f154eb826071209 Mon Sep 17 00:00:00 2001 From: XU Mingshi <91017482+mxuax@users.noreply.github.com> Date: Fri, 30 Jan 2026 21:44:58 +0800 Subject: [PATCH 75/83] [BugFix] Fix modulate_index shape error in Qwen-Image-Edit Task (#1100) Signed-off-by: mxuax Signed-off-by: XU Mingshi <91017482+mxuax@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- .../qwen_image/qwen_image_transformer.py | 84 +++++++++++++++++-- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py index 47098ff8a..a3cc6cbbb 100644 --- a/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py +++ b/vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py @@ -89,6 +89,66 @@ class ImageRopePrepare(nn.Module): return hidden_states, vid_freqs, txt_freqs +class ModulateIndexPrepare(nn.Module): + """Prepares modulate_index for sequence parallel when zero_cond_t is enabled. + + This module encapsulates the creation of modulate_index tensor, which is used + to select different conditioning parameters (shift/scale/gate) for different + token positions in image editing tasks. + + Similar to Z-Image's UnifiedPrepare and ImageRopePrepare, this creates a module + boundary where _sp_plan can shard the output via split_output=True. + + The modulate_index must be sharded along the sequence dimension to match the + sharded hidden_states in SP mode. + + Note: Our _sp_plan corresponds to diffusers' _cp_plan (Context Parallelism). + """ + + def __init__(self, zero_cond_t: bool = False): + super().__init__() + self.zero_cond_t = zero_cond_t + + def forward( + self, + timestep: torch.Tensor, + img_shapes: list[list[tuple[int, int, int]]], + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Prepare timestep and modulate_index for SP. + + Args: + timestep: Timestep tensor [batch] + img_shapes: List of image shape tuples per batch item. + Each item is a list of (frame, height, width) tuples. + For edit models: [[source_shape], [target_shape1, target_shape2, ...]] + + Returns: + timestep: Doubled timestep if zero_cond_t, else original [batch] or [2*batch] + modulate_index: Token condition index [batch, seq_len] if zero_cond_t, else None + - index=0: source image tokens (use normal timestep conditioning) + - index=1: target image tokens (use zero timestep conditioning) + + Note: _sp_plan will shard modulate_index via split_output=True when SP is enabled. + The modulate_index sequence dimension must match hidden_states after sharding. + """ + if self.zero_cond_t: + # Double the timestep: [timestep, timestep * 0] + # This creates two sets of conditioning parameters in AdaLayerNorm + timestep = torch.cat([timestep, timestep * 0], dim=0) + + # Create modulate_index to select conditioning per token position + # - First image (sample[0]): source image, use index=0 (normal timestep) + # - Remaining images (sample[1:]): target images, use index=1 (zero timestep) + modulate_index = torch.tensor( + [[0] * prod(sample[0]) + [1] * sum([prod(s) for s in sample[1:]]) for sample in img_shapes], + device=timestep.device, + dtype=torch.int, + ) + return timestep, modulate_index + + return timestep, None + + class QwenTimestepProjEmbeddings(nn.Module): def __init__(self, embedding_dim, use_additional_t_cond=False): super().__init__() @@ -785,6 +845,12 @@ class QwenImageTransformer2DModel(CachedTransformer): 1: SequenceParallelInput(split_dim=0, expected_dims=2, split_output=True, auto_pad=True), # txt_freqs (index 2) is NOT sharded - kept replicated for dual-stream attention }, + # Shard ModulateIndexPrepare output (modulate_index must be sharded to match hidden_states) + # This is only active when zero_cond_t=True (image editing models) + # Output index 1 is modulate_index [batch, seq_len], needs sharding along dim=1 + "modulate_index_prepare": { + 1: SequenceParallelInput(split_dim=1, expected_dims=2, split_output=True, auto_pad=True), + }, # Gather output at proj_out "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3), } @@ -848,6 +914,11 @@ class QwenImageTransformer2DModel(CachedTransformer): # This ensures RoPE dimensions align with hidden_states after sharding self.image_rope_prepare = ImageRopePrepare(self.img_in, self.pos_embed) + # ModulateIndexPrepare module for _sp_plan to shard modulate_index + # This ensures modulate_index dimensions align with hidden_states after sharding + # Only active when zero_cond_t=True (image editing models) + self.modulate_index_prepare = ModulateIndexPrepare(zero_cond_t=zero_cond_t) + def forward( self, hidden_states: torch.Tensor, @@ -906,15 +977,10 @@ class QwenImageTransformer2DModel(CachedTransformer): # Ensure timestep tensor is on the same device and dtype as hidden_states timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype) - if self.zero_cond_t: - timestep = torch.cat([timestep, timestep * 0], dim=0) - modulate_index = torch.tensor( - [[0] * prod(sample[0]) + [1] * sum([prod(s) for s in sample[1:]]) for sample in img_shapes], - device=timestep.device, - dtype=torch.int, - ) - else: - modulate_index = None + # Prepare timestep and modulate_index via ModulateIndexPrepare module + # _sp_plan will shard modulate_index via split_output=True (when zero_cond_t=True) + # This ensures modulate_index sequence dimension matches sharded hidden_states + timestep, modulate_index = self.modulate_index_prepare(timestep, img_shapes) encoder_hidden_states = self.txt_norm(encoder_hidden_states) encoder_hidden_states = self.txt_in(encoder_hidden_states) From 4eeea68cff26c0f1101f95868305a716b6cd25a5 Mon Sep 17 00:00:00 2001 From: Canlin Guo Date: Fri, 30 Jan 2026 23:21:43 +0800 Subject: [PATCH 76/83] [Platform] Add supports_torch_inductor interface (#1108) Signed-off-by: gcanlin --- .../worker/diffusion_model_runner.py | 21 ++++++++++++------- vllm_omni/platforms/cuda/platform.py | 4 ++++ vllm_omni/platforms/interface.py | 5 +++++ vllm_omni/platforms/npu/platform.py | 4 ++++ vllm_omni/platforms/rocm/platform.py | 4 ++++ vllm_omni/platforms/xpu/platform.py | 4 ++++ 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/vllm_omni/diffusion/worker/diffusion_model_runner.py b/vllm_omni/diffusion/worker/diffusion_model_runner.py index b0bec0852..38f438062 100644 --- a/vllm_omni/diffusion/worker/diffusion_model_runner.py +++ b/vllm_omni/diffusion/worker/diffusion_model_runner.py @@ -29,6 +29,7 @@ from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineL from vllm_omni.diffusion.offload import apply_offload_hooks from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.distributed.omni_connectors.kv_transfer_manager import OmniKVTransferManager +from vllm_omni.platforms import current_omni_platform logger = init_logger(__name__) @@ -112,14 +113,20 @@ class DiffusionModelRunner: # Apply torch.compile if not in eager mode if not self.od_config.enforce_eager: - try: - self.pipeline.transformer = regionally_compile( - self.pipeline.transformer, - dynamic=True, + if current_omni_platform.supports_torch_inductor(): + try: + self.pipeline.transformer = regionally_compile( + self.pipeline.transformer, + dynamic=True, + ) + logger.info("Model runner: Model compiled with torch.compile.") + except Exception as e: + logger.warning(f"Model runner: torch.compile failed with error: {e}. Using eager mode.") + else: + logger.warning( + "Model runner: Platform %s does not support torch inductor, skipping torch.compile.", + current_omni_platform.get_torch_device(), ) - logger.info("Model runner: Model compiled with torch.compile.") - except Exception as e: - logger.warning(f"Model runner: torch.compile failed with error: {e}. Using eager mode.") # Setup cache backend self.cache_backend = get_cache_backend(self.od_config.cache_backend, self.od_config.cache_config) diff --git a/vllm_omni/platforms/cuda/platform.py b/vllm_omni/platforms/cuda/platform.py index 89e403f99..6bf740a01 100644 --- a/vllm_omni/platforms/cuda/platform.py +++ b/vllm_omni/platforms/cuda/platform.py @@ -80,6 +80,10 @@ class CudaOmniPlatform(OmniPlatform, CudaPlatformBase): logger.info("Defaulting to diffusion attention backend SDPA") return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + @classmethod + def supports_torch_inductor(cls) -> bool: + return True + @classmethod def get_torch_device(cls, local_rank: int | None = None) -> torch.device: if local_rank is None: diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py index db541ebce..a04224063 100644 --- a/vllm_omni/platforms/interface.py +++ b/vllm_omni/platforms/interface.py @@ -73,6 +73,11 @@ class OmniPlatform(Platform): """ raise NotImplementedError + @classmethod + def supports_torch_inductor(cls) -> bool: + """Check if the platform supports torch.compile with inductor backend.""" + raise NotImplementedError + @classmethod def get_torch_device(cls, local_rank: int | None = None) -> torch.device: raise NotImplementedError diff --git a/vllm_omni/platforms/npu/platform.py b/vllm_omni/platforms/npu/platform.py index 429297bbd..3c2495c3d 100644 --- a/vllm_omni/platforms/npu/platform.py +++ b/vllm_omni/platforms/npu/platform.py @@ -55,6 +55,10 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform): logger.info("Falling back to diffusion attention backend SDPA") return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + @classmethod + def supports_torch_inductor(cls) -> bool: + return False + @classmethod def get_torch_device(cls, local_rank: int | None = None) -> torch.device: if local_rank is None: diff --git a/vllm_omni/platforms/rocm/platform.py b/vllm_omni/platforms/rocm/platform.py index f2c2d3a85..14534f3a1 100644 --- a/vllm_omni/platforms/rocm/platform.py +++ b/vllm_omni/platforms/rocm/platform.py @@ -66,6 +66,10 @@ class RocmOmniPlatform(OmniPlatform, RocmPlatform): logger.info("Defaulting to diffusion attention backend SDPA") return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + @classmethod + def supports_torch_inductor(cls) -> bool: + return True + @classmethod def get_default_stage_config_path(cls) -> str: return "vllm_omni/platforms/rocm/stage_configs" diff --git a/vllm_omni/platforms/xpu/platform.py b/vllm_omni/platforms/xpu/platform.py index 5c126c6d9..43d635702 100644 --- a/vllm_omni/platforms/xpu/platform.py +++ b/vllm_omni/platforms/xpu/platform.py @@ -43,6 +43,10 @@ class XPUOmniPlatform(OmniPlatform, XPUPlatform): logger.info("Defaulting to diffusion attention backend SDPA") return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + @classmethod + def supports_torch_inductor(cls) -> bool: + return True + @classmethod def get_default_stage_config_path(cls) -> str: return "vllm_omni/platforms/xpu/stage_configs" From f6cfc0d1e398715e6061f1ad12238971a0dbe7ac Mon Sep 17 00:00:00 2001 From: Ziming Huang Date: Fri, 30 Jan 2026 23:25:49 +0800 Subject: [PATCH 77/83] [BugFix] Fix Qwen3 Omni talker mtp torch.compile startup error (#1104) Signed-off-by: ram16g Signed-off-by: ZeldaHuang Co-authored-by: ram16g Co-authored-by: Hongsheng Liu --- tests/worker/test_omni_gpu_model_runner.py | 7 +++++++ .../qwen3_omni_moe_code_predictor_mtp.py | 12 +++-------- .../qwen3_omni/qwen3_omni_moe_talker.py | 5 ++++- vllm_omni/worker/gpu_model_runner.py | 20 ++++++++++--------- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/tests/worker/test_omni_gpu_model_runner.py b/tests/worker/test_omni_gpu_model_runner.py index a600e350a..eb1adf227 100644 --- a/tests/worker/test_omni_gpu_model_runner.py +++ b/tests/worker/test_omni_gpu_model_runner.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +from types import SimpleNamespace import torch @@ -86,6 +87,12 @@ def test_talker_mtp_forward_cpu_updates_inputs_and_info(monkeypatch): runner = _make_runner(req_ids=("r1", "r2"), hidden_size=4) + def fake_determine(self, num_tokens, num_reqs, num_scheduled_tokens_np, max_num_scheduled_tokens, use_cascade_attn): + batch_desc = SimpleNamespace(num_tokens=int(num_tokens)) + return (False, batch_desc, None, None, None) + + monkeypatch.setattr(runner, "_determine_batch_execution_and_padding", fake_determine.__get__(runner, type(runner))) + # Initialize per-request embeds (batch-major inside talker_mtp_inputs_embeds) runner.talker_mtp_inputs_embeds.gpu[0] = torch.tensor([1.0, 2.0, 3.0, 4.0]) runner.talker_mtp_inputs_embeds.gpu[1] = torch.tensor([10.0, 20.0, 30.0, 40.0]) diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py index 6c135926d..fc7402890 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py @@ -458,10 +458,6 @@ class Qwen3OmniMoeTalkerCodePredictor(nn.Module): TopPLogitsWarper(top_p=0.8), ] ) - max_batch_size = vllm_config.scheduler_config.max_num_seqs - self.position_ids_buffer = [] - for layer_idx in range(self.num_code_groups - 1): - self.position_ids_buffer.append(torch.arange(layer_idx + 2, dtype=torch.int64).repeat(max_batch_size)) compilation_config = get_current_vllm_config().compilation_config if prefix in compilation_config.static_forward_context: @@ -508,13 +504,11 @@ class Qwen3OmniMoeTalkerCodePredictor(nn.Module): batch_size = current_input.shape[0] # Predict all residual layers (layers 1 to num_code_groups-1) autoregressively - begin_pos = 0 for layer_idx in range(self.num_code_groups - 1): - # Input for this layer: [last_talker_hidden, prev_layer_embed] - + seq_len = layer_idx + 2 + # Compute position_ids dynamically to avoid torch.compile specializing batch_size + position_ids = torch.arange(seq_len, device=current_input.device, dtype=torch.int64).repeat(batch_size) # Forward through code_predictor model - position_ids = self.position_ids_buffer[layer_idx][: (layer_idx + 2) * batch_size] # [1, seq_len] - begin_pos += layer_idx + 2 outputs = self.model( inputs_embeds=current_input, attention_mask=None, diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py index 65cb43d0b..946694ed2 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py @@ -135,8 +135,11 @@ class Qwen3OmniMoeTalkerForConditionalGeneration( self.code_predictor = Qwen3OmniMoeTalkerCodePredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "code_predictor") ) + max_batch_size = max( + vllm_config.scheduler_config.max_num_seqs, vllm_config.compilation_config.max_cudagraph_capture_size + ) self.layer0_embed_buffer = torch.zeros( - (vllm_config.scheduler_config.max_num_seqs, 1, self.config.text_config.hidden_size), + (max_batch_size, 1, self.config.text_config.hidden_size), dtype=vllm_config.model_config.dtype, ) diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index 1e2beb046..b3d1833f2 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -52,12 +52,13 @@ class OmniGPUModelRunner(GPUModelRunner): self.model.talker_mtp, self.vllm_config, runtime_mode=CUDAGraphMode.FULL ) hidden_size = self.model_config.hf_config.talker_config.text_config.hidden_size - self.talker_mtp_input_ids = self._make_buffer(self.max_num_reqs, dtype=torch.int32) + max_batch_size = max(self.max_num_reqs, self.compilation_config.max_cudagraph_capture_size) + self.talker_mtp_input_ids = self._make_buffer(max_batch_size, dtype=torch.int32) self.talker_mtp_inputs_embeds = self._make_buffer( - self.max_num_reqs, hidden_size, dtype=self.dtype, numpy=False + max_batch_size, hidden_size, dtype=self.dtype, numpy=False ) - self.last_talker_hidden = self._make_buffer(self.max_num_reqs, hidden_size, dtype=self.dtype, numpy=False) - self.text_step = self._make_buffer(self.max_num_reqs, hidden_size, dtype=self.dtype, numpy=False) + self.last_talker_hidden = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False) + self.text_step = self._make_buffer(max_batch_size, hidden_size, dtype=self.dtype, numpy=False) def _init_mrope_positions(self, req_state: CachedRequestState): """Initialize M-RoPE positions for multimodal inputs. @@ -645,7 +646,7 @@ class OmniGPUModelRunner(GPUModelRunner): if getattr(self.model, "talker", None) is not None and hasattr(self.model, "talker_mtp"): num_tokens_padded_talker_mtp = num_tokens_padded if num_tokens_padded_talker_mtp == self.max_num_tokens: - num_tokens_padded_talker_mtp = min(num_tokens_padded, self.max_num_reqs) + num_tokens_padded_talker_mtp = self.talker_mtp_input_ids.gpu.shape[0] outputs = self.talker_mtp( self.talker_mtp_input_ids.gpu[:num_tokens_padded_talker_mtp], self.talker_mtp_inputs_embeds.gpu[:num_tokens_padded_talker_mtp], @@ -1075,10 +1076,11 @@ class OmniGPUModelRunner(GPUModelRunner): max_num_scheduled_tokens=1, use_cascade_attn=False, ) - req_input_ids = self.talker_mtp_input_ids.gpu[:decode_batch_size] - req_embeds = self.talker_mtp_inputs_embeds.gpu[:decode_batch_size] - last_talker_hidden = self.last_talker_hidden.gpu[:decode_batch_size] - text_step = self.text_step.gpu[:decode_batch_size] + num_tokens_padded = batch_desc.num_tokens + req_input_ids = self.talker_mtp_input_ids.gpu[:num_tokens_padded] + req_embeds = self.talker_mtp_inputs_embeds.gpu[:num_tokens_padded] + last_talker_hidden = self.last_talker_hidden.gpu[:num_tokens_padded] + text_step = self.text_step.gpu[:num_tokens_padded] with set_forward_context( None, self.vllm_config, cudagraph_runtime_mode=_cudagraph_mode, batch_descriptor=batch_desc ): From 8ad39570b8c4286f505edcfb5cd0a265b179f2ca Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Sat, 31 Jan 2026 00:05:34 +0800 Subject: [PATCH 78/83] [Bugfix] fix request_id of image generation in api server (#1112) Signed-off-by: zjy0516 Co-authored-by: Hongsheng Liu --- .buildkite/pipeline.yml | 2 +- ..._image_input.py => test_image_gen_edit.py} | 109 ++++++++++++++---- vllm_omni/entrypoints/openai/api_server.py | 3 +- 3 files changed, 89 insertions(+), 25 deletions(-) rename tests/e2e/online_serving/{test_i2i_multi_image_input.py => test_image_gen_edit.py} (66%) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 49d1b2ece..25b14ce67 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -240,7 +240,7 @@ steps: depends_on: image-build commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/online_serving/test_i2i_multi_image_input.py + - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py agents: queue: "mithril-h100-pool" plugins: diff --git a/tests/e2e/online_serving/test_i2i_multi_image_input.py b/tests/e2e/online_serving/test_image_gen_edit.py similarity index 66% rename from tests/e2e/online_serving/test_i2i_multi_image_input.py rename to tests/e2e/online_serving/test_image_gen_edit.py index b1fcac1de..e796fe959 100644 --- a/tests/e2e/online_serving/test_i2i_multi_image_input.py +++ b/tests/e2e/online_serving/test_image_gen_edit.py @@ -10,11 +10,14 @@ import signal import socket import subprocess import sys +import threading import time from io import BytesIO +from typing import Any import openai import pytest +import requests from PIL import Image from vllm.assets.image import ImageAsset from vllm.utils.network_utils import get_open_port @@ -23,6 +26,7 @@ os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" models = ["Qwen/Qwen-Image-Edit-2509"] test_params = models +t2i_models = ["Tongyi-MAI/Z-Image-Turbo"] class OmniServer: @@ -174,35 +178,94 @@ def _decode_data_url_to_image_bytes(data_url: str) -> bytes: @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_i2i_multi_image_input_qwen_image_edit_2509( - client: openai.OpenAI, omni_server, base64_encoded_images: list[str], ) -> None: - """Test multi-image input editing via OpenAI API.""" + """Test multi-image input editing via OpenAI API with concurrent requests.""" image_data_urls = [f"data:image/png;base64,{img}" for img in base64_encoded_images] messages = dummy_messages_from_image_data(image_data_urls) - height = 832 - width = 1248 - chat_completion = client.chat.completions.create( - model=omni_server.model, - messages=messages, - extra_body={ - "height": height, - "width": width, + barrier = threading.Barrier(2) + results: list[tuple[int, int]] = [] + + def _call_chat(width: int, height: int) -> None: + client = openai.OpenAI( + base_url=f"http://{omni_server.host}:{omni_server.port}/v1", + api_key="EMPTY", + ) + barrier.wait() + chat_completion = client.chat.completions.create( + model=omni_server.model, + messages=messages, + extra_body={ + "height": height, + "width": width, + "num_inference_steps": 2, + "guidance_scale": 0.0, + "seed": 42, + }, + ) + + assert len(chat_completion.choices) == 1 + choice = chat_completion.choices[0] + assert choice.finish_reason == "stop" + assert choice.message.role == "assistant" + + image_data_url = _extract_image_data_url(choice.message.content) + image_bytes = _decode_data_url_to_image_bytes(image_data_url) + img = Image.open(BytesIO(image_bytes)) + img.load() + results.append(img.size) + + threads = [ + threading.Thread(target=_call_chat, args=(1248, 832)), + threading.Thread(target=_call_chat, args=(1024, 768)), + ] + for t in threads: + t.start() + for t in threads: + t.join() + + # TODO @ZJY + # assert (1248, 832) in results + # assert (1024, 768) in results + + +@pytest.mark.parametrize("omni_server", t2i_models, indirect=True) +def test_t2i_concurrent_requests_different_sizes(omni_server) -> None: + """Test /v1/images/generations concurrent requests with different sizes.""" + base_url = f"http://{omni_server.host}:{omni_server.port}" + url = f"{base_url}/v1/images/generations" + + barrier = threading.Barrier(2) + results: list[tuple[int, int]] = [] + + def _call_generate(size: str) -> None: + payload: dict[str, Any] = { + "prompt": "cute cat playing with a ball", + "n": 1, + "size": size, + "response_format": "b64_json", "num_inference_steps": 2, - "guidance_scale": 0.0, - "seed": 42, - }, - ) + } + barrier.wait() + response = requests.post(url, json=payload, timeout=120) + assert response.status_code == 200 + data = response.json() + image_b64 = data["data"][0]["b64_json"] + image_bytes = base64.b64decode(image_b64) + img = Image.open(BytesIO(image_bytes)) + img.load() + results.append(img.size) - assert len(chat_completion.choices) == 1 - choice = chat_completion.choices[0] - assert choice.finish_reason == "stop" - assert choice.message.role == "assistant" + threads = [ + threading.Thread(target=_call_generate, args=("512x512",)), + threading.Thread(target=_call_generate, args=("768x512",)), + ] + for t in threads: + t.start() + for t in threads: + t.join() - image_data_url = _extract_image_data_url(choice.message.content) - image_bytes = _decode_data_url_to_image_bytes(image_data_url) - img = Image.open(BytesIO(image_bytes)) - img.load() - assert img.size == (width, height) + assert (512, 512) in results + assert (768, 512) in results diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index aedaceee3..556ed8cfe 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -7,6 +7,7 @@ import os # Image generation API imports import random import time +import uuid from argparse import Namespace from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -995,7 +996,7 @@ async def generate_images(request: ImageGenerationRequest, raw_request: Request) # might produce blurry images in some environments. gen_params.seed = random.randint(0, 2**32 - 1) - request_id = f"img_gen_{int(time.time())}" + request_id = f"img_gen_{uuid.uuid4().hex}" logger.info(f"Generating {request.n} image(s) {size_str}") From 23cf48d7bb5c49b4da68febc26637b3c3f1caf77 Mon Sep 17 00:00:00 2001 From: Didan Deng <33117903+wtomin@users.noreply.github.com> Date: Sat, 31 Jan 2026 00:23:44 +0800 Subject: [PATCH 79/83] [Perf]: CFG parallel abstraction (#851) Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com> Co-authored-by: Hongsheng Liu --- .../diffusion/parallelism_acceleration.md | 197 +++++--- docs/user_guide/diffusion_acceleration.md | 14 +- .../offline_inference/image_to_image.md | 1 + .../offline_inference/image_to_video.md | 1 + .../offline_inference/text_to_image.md | 1 + .../offline_inference/text_to_video.md | 1 + .../image_to_image/image_to_image.md | 1 + .../image_to_video/README.md | 1 + .../image_to_video/image_to_video.py | 29 +- .../offline_inference/text_to_image/README.md | 1 + .../text_to_video/text_to_video.md | 1 + .../text_to_video/text_to_video.py | 13 +- .../distributed/test_cfg_parallel.py | 423 ++++++++++++++++++ vllm_omni/diffusion/data.py | 1 + .../diffusion/distributed/cfg_parallel.py | 235 ++++++++++ .../diffusion/models/flux/pipeline_flux.py | 18 + .../flux2_klein/pipeline_flux2_klein.py | 67 +-- .../longcat_image/pipeline_longcat_image.py | 79 ++-- .../pipeline_longcat_image_edit.py | 62 +-- .../models/ovis_image/pipeline_ovis_image.py | 149 +++--- .../diffusion/models/qwen_image/__init__.py | 4 + .../models/qwen_image/cfg_parallel.py | 160 +++++++ .../models/qwen_image/pipeline_qwen_image.py | 123 +---- .../qwen_image/pipeline_qwen_image_edit.py | 134 +----- .../pipeline_qwen_image_edit_plus.py | 129 +----- .../qwen_image/pipeline_qwen_image_layered.py | 154 +------ .../diffusion/models/sd3/pipeline_sd3.py | 85 ++-- .../models/wan2_2/pipeline_wan2_2.py | 73 ++- .../models/wan2_2/pipeline_wan2_2_i2v.py | 80 +++- .../models/wan2_2/pipeline_wan2_2_ti2v.py | 76 +++- vllm_omni/entrypoints/cli/serve.py | 7 +- 31 files changed, 1506 insertions(+), 814 deletions(-) create mode 100644 tests/diffusion/distributed/test_cfg_parallel.py create mode 100644 vllm_omni/diffusion/distributed/cfg_parallel.py create mode 100644 vllm_omni/diffusion/models/qwen_image/cfg_parallel.py diff --git a/docs/user_guide/diffusion/parallelism_acceleration.md b/docs/user_guide/diffusion/parallelism_acceleration.md index 1a9f540ad..7334df1d4 100644 --- a/docs/user_guide/diffusion/parallelism_acceleration.md +++ b/docs/user_guide/diffusion/parallelism_acceleration.md @@ -373,9 +373,9 @@ def forward(self, hidden_states, ...): ### CFG-Parallel -##### Offline Inference +#### Offline Inference -CFG-Parallel is enabled through `DiffusionParallelConfig(cfg_parallel_size=...)`. The recommended configuration is `cfg_parallel_size=2` (one rank for the positive branch and one rank for the negative branch). +CFG-Parallel is enabled through `DiffusionParallelConfig(cfg_parallel_size=2)`, which runs one rank for the positive branch and one rank for the negative branch. An example of offline inference using CFG-Parallel (image-to-image) is shown below: @@ -383,19 +383,21 @@ An example of offline inference using CFG-Parallel (image-to-image) is shown bel from vllm_omni import Omni from vllm_omni.diffusion.data import DiffusionParallelConfig +image_path = "path_to_image.png" omni = Omni( model="Qwen/Qwen-Image-Edit", parallel_config=DiffusionParallelConfig(cfg_parallel_size=2), ) +input_image = Image.open(image_path).convert("RGB") outputs = omni.generate( { "prompt": "turn this cat to a dog", "negative_prompt": "low quality, blurry", + "multi_modal_data": {"image": input_image}, }, OmniDiffusionSamplingParams( true_cfg_scale=4.0, - pil_image=input_image, num_inference_steps=50, ), ) @@ -403,7 +405,28 @@ outputs = omni.generate( Notes: -- CFG-Parallel is only effective when **true CFG** is enabled (i.e., `true_cfg_scale > 1` and a `negative_prompt` is provided). +- CFG-Parallel is only effective when a `negative_prompt` is provided AND a guidance scale (or `cfg_scale`) is greater than 1. + +See `examples/offline_inference/image_to_image/image_edit.py` for a complete working example. +```bash +cd examples/offline_inference/image_to_image/ +python image_edit.py \ + --model "Qwen/Qwen-Image-Edit" \ + --image "qwen_image_output.png" \ + --prompt "turn this cat to a dog" \ + --negative_prompt "low quality, blurry" \ + --cfg_scale 4.0 \ + --output "edited_image.png" \ + --cfg_parallel_size 2 +``` + +#### Online Serving + +You can enable CFG-Parallel in online serving for diffusion models via `--cfg-parallel-size`: + +```bash +vllm serve Qwen/Qwen-Image-Edit --omni --port 8091 --cfg-parallel-size 2 +``` #### How to parallelize a pipeline @@ -416,58 +439,130 @@ In `QwenImagePipeline`, each diffusion step runs two denoiser forward passes seq CFG-Parallel assigns these two branches to different ranks in the **CFG group** and synchronizes the results. -Below is an example of CFG-Parallel implementation: +vLLM-omni provides `CFGParallelMixin` base class that encapsulates the CFG parallel logic. By inheriting from this mixin and calling its methods, pipelines can easily implement CFG parallel without writing repetitive code. + +**Key Methods in CFGParallelMixin:** +- `predict_noise_maybe_with_cfg()`: Automatically handles CFG parallel noise prediction +- `scheduler_step_maybe_with_cfg()`: Scheduler step with automatic CFG rank synchronization + +**Example Implementation:** ```python -def diffuse( +class QwenImageCFGParallelMixin(CFGParallelMixin): + """ + Base Mixin class for Qwen Image pipelines providing shared CFG methods. + """ + + def diffuse( self, - ... - ): - # Enable CFG-parallel: rank0 computes positive, rank1 computes negative. - cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1 + prompt_embeds: torch.Tensor, + prompt_embeds_mask: torch.Tensor, + negative_prompt_embeds: torch.Tensor, + negative_prompt_embeds_mask: torch.Tensor, + latents: torch.Tensor, + img_shapes: torch.Tensor, + txt_seq_lens: torch.Tensor, + negative_txt_seq_lens: torch.Tensor, + timesteps: torch.Tensor, + do_true_cfg: bool, + guidance: torch.Tensor, + true_cfg_scale: float, + image_latents: torch.Tensor | None = None, + cfg_normalize: bool = True, + additional_transformer_kwargs: dict[str, Any] | None = None, + ) -> torch.Tensor: + self.transformer.do_true_cfg = do_true_cfg - self.transformer.do_true_cfg = do_true_cfg + for i, t in enumerate(timesteps): + timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) - if cfg_parallel_ready: - cfg_group = get_cfg_group() - cfg_rank = get_classifier_free_guidance_rank() + # Prepare kwargs for positive (conditional) prediction + positive_kwargs = { + "hidden_states": latents, + "timestep": timestep / 1000, + "guidance": guidance, + "encoder_hidden_states_mask": prompt_embeds_mask, + "encoder_hidden_states": prompt_embeds, + "img_shapes": img_shapes, + "txt_seq_lens": txt_seq_lens, + } - if cfg_rank == 0: - local_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - else: - local_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] + # Prepare kwargs for negative (unconditional) prediction + if do_true_cfg: + negative_kwargs = { + "hidden_states": latents, + "timestep": timestep / 1000, + "guidance": guidance, + "encoder_hidden_states_mask": negative_prompt_embeds_mask, + "encoder_hidden_states": negative_prompt_embeds, + "img_shapes": img_shapes, + "txt_seq_lens": negative_txt_seq_lens, + } + else: + negative_kwargs = None - gathered = cfg_group.all_gather(local_pred, separate_tensors=True) - if cfg_rank == 0: - noise_pred = gathered[0] - neg_noise_pred = gathered[1] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - cfg_group.broadcast(latents, src=0) - else: - # fallback: run positive then negative sequentially on one rank - ... + # Predict noise with automatic CFG parallel handling + # - In CFG parallel mode: rank0 computes positive, rank1 computes negative + # - Automatically gathers results and combines them on rank0 + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=true_cfg_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=cfg_normalize, + ) + + # Step scheduler with automatic CFG synchronization + # - Only rank0 computes the scheduler step + # - Automatically broadcasts updated latents to all ranks + latents = self.scheduler_step_maybe_with_cfg( + noise_pred, t, latents, do_true_cfg + ) + + return latents +``` + +**How it works:** +1. Prepare separate `positive_kwargs` and `negative_kwargs` for conditional and unconditional predictions +2. Call `predict_noise_maybe_with_cfg()` which: + - Detects if CFG parallel is enabled (`get_classifier_free_guidance_world_size() > 1`) + - Distributes computation: rank0 processes positive, rank1 processes negative + - Gathers predictions and combines them using `combine_cfg_noise()` on rank0 + - Returns combined noise prediction (only valid on rank0) +3. Call `scheduler_step_maybe_with_cfg()` which: + - Only rank0 computes the scheduler step + - Broadcasts the updated latents to all ranks for synchronization + +**How to customize** + +Some pipelines may need to customize the following functions in `CFGParallelMixin`: +1. You may need to edit `predict_noise` function for custom behaviors. +```python +def predict_noise(self, *args, **kwargs): + """ + Forward pass through transformer to predict noise. + + Subclasses should override this if they need custom behavior, + but the default implementation calls self.transformer. + """ + return self.transformer(*args, **kwargs)[0] + +``` +2. The default normalization function after combining the noise predictions from both branches is as follows. You may need to customize it. +```python +def cfg_normalize_function(self, noise_pred, comb_pred): + """ + Normalize the combined noise prediction. + + Args: + noise_pred: positive noise prediction + comb_pred: combined noise prediction after CFG + + Returns: + Normalized noise prediction tensor + """ + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) + noise_pred = comb_pred * (cond_norm / noise_norm) + return noise_pred ``` diff --git a/docs/user_guide/diffusion_acceleration.md b/docs/user_guide/diffusion_acceleration.md index 8d31747d2..d08124378 100644 --- a/docs/user_guide/diffusion_acceleration.md +++ b/docs/user_guide/diffusion_acceleration.md @@ -39,23 +39,23 @@ The following table shows which models are currently supported by each accelerat | Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention | CFG-Parallel | |-------|------------------|:----------:|:-----------:|:-----------:|:----------------:|:----------------:| -| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ✅ | ❌ | ❌ | ❌ | -| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ✅ | ❌ | ❌ | ❌ | -| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ✅ | ❌ | ❌ | ❌ | +| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ✅ | ❌ | ❌ | ✅ | +| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ✅ | ❌ | ❌ | ✅ | +| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ✅ | ❌ | ❌ | ✅ | | **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ | ✅ | ✅ | | **Qwen-Image-2512** | `Qwen/Qwen-Image-2512` | ✅ | ✅ | ✅ | ✅ | ✅ | | **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ | ✅ | ✅ | ✅ | | **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ✅ | ✅ | ✅ | ✅ | ✅ | | **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ❌ | ✅ | ✅ | ✅ | ✅ | | **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ✅ | ✅ | ❌ | ❌ | ❌ | -| **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ✅ | ❌ | ❌ | ❌ | +| **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ✅ | ❌ | ❌ | ✅ | | **Bagel** | `ByteDance-Seed/BAGEL-7B-MoT` | ✅ | ✅ | ❌ | ❌ | ❌ | ### VideoGen -| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention | -|-------|------------------|:--------:|:---------:|:----------:|:--------------:| -| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ❌ | ✅ | ❌ | ❌ | +| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention |CFG-Parallel | +|-------|------------------|:--------:|:---------:|:----------:|:--------------:|:----------------:| +| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ❌ | ✅ | ✅ | ✅ | ✅ | ## Performance Benchmarks diff --git a/docs/user_guide/examples/offline_inference/image_to_image.md b/docs/user_guide/examples/offline_inference/image_to_image.md index 78c7f11fc..c970106d2 100644 --- a/docs/user_guide/examples/offline_inference/image_to_image.md +++ b/docs/user_guide/examples/offline_inference/image_to_image.md @@ -47,6 +47,7 @@ Key arguments: - `--image`: path(s) to the source image(s) (PNG/JPG, converted to RGB). Can specify multiple images. - `--prompt` / `--negative_prompt`: text description (string). - `--cfg_scale`: true classifier-free guidance scale (default: 4.0). Classifier-free guidance is enabled by setting cfg_scale > 1 and providing a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, usually at the expense of lower image quality. +- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. - `--guidance_scale`: guidance scale for guidance-distilled models (default: 1.0, disabled). Unlike classifier-free guidance (--cfg_scale), guidance-distilled models take the guidance scale directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models. - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--output`: path to save the generated PNG. diff --git a/docs/user_guide/examples/offline_inference/image_to_video.md b/docs/user_guide/examples/offline_inference/image_to_video.md index 32c655ffd..d65839dd7 100644 --- a/docs/user_guide/examples/offline_inference/image_to_video.md +++ b/docs/user_guide/examples/offline_inference/image_to_video.md @@ -52,6 +52,7 @@ Key arguments: - `--num_frames`: Number of frames (default 81). - `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high-noise stages for MoE). - `--negative_prompt`: Optional list of artifacts to suppress. +- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. - `--boundary_ratio`: Boundary split ratio for two-stage MoE models. - `--flow_shift`: Scheduler flow shift (5.0 for 720p, 12.0 for 480p). - `--num_inference_steps`: Number of denoising steps (default 50). diff --git a/docs/user_guide/examples/offline_inference/text_to_image.md b/docs/user_guide/examples/offline_inference/text_to_image.md index 2dd0f7b7c..486b9f63b 100644 --- a/docs/user_guide/examples/offline_inference/text_to_image.md +++ b/docs/user_guide/examples/offline_inference/text_to_image.md @@ -95,6 +95,7 @@ Key arguments: - `--prompt`: text description (string). - `--seed`: integer seed for deterministic sampling. - `--cfg_scale`: true CFG scale (model-specific guidance strength). +- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. - `--num_images_per_prompt`: number of images to generate per prompt (saves as `output`, `output_1`, ...). - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--height/--width`: output resolution (defaults 1024x1024). diff --git a/docs/user_guide/examples/offline_inference/text_to_video.md b/docs/user_guide/examples/offline_inference/text_to_video.md index 0cc22edea..db0860b38 100644 --- a/docs/user_guide/examples/offline_inference/text_to_video.md +++ b/docs/user_guide/examples/offline_inference/text_to_video.md @@ -28,6 +28,7 @@ Key arguments: - `--num_frames`: Number of frames (Wan default is 81). - `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high).. - `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string). +- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. - `--boundary_ratio`: Boundary split ratio for low/high DiT. - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: path to save the generated video. diff --git a/examples/offline_inference/image_to_image/image_to_image.md b/examples/offline_inference/image_to_image/image_to_image.md index b9e25bf98..d0986d6ee 100644 --- a/examples/offline_inference/image_to_image/image_to_image.md +++ b/examples/offline_inference/image_to_image/image_to_image.md @@ -49,6 +49,7 @@ Key arguments: - `--output`: path to save the generated PNG. - `--vae_use_slicing`: enable VAE slicing for memory optimization. - `--vae_use_tiling`: enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). - `--enable-cpu-offload`: enable CPU offloading for diffusion models. > ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. diff --git a/examples/offline_inference/image_to_video/README.md b/examples/offline_inference/image_to_video/README.md index 52bb389e7..a1355dab6 100644 --- a/examples/offline_inference/image_to_video/README.md +++ b/examples/offline_inference/image_to_video/README.md @@ -56,6 +56,7 @@ Key arguments: - `--output`: Path to save the generated video. - `--vae_use_slicing`: Enable VAE slicing for memory optimization. - `--vae_use_tiling`: Enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). - `--enable-cpu-offload`: enable CPU offloading for diffusion models. > ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. diff --git a/examples/offline_inference/image_to_video/image_to_video.py b/examples/offline_inference/image_to_video/image_to_video.py index 178528784..8e8d39915 100644 --- a/examples/offline_inference/image_to_video/image_to_video.py +++ b/examples/offline_inference/image_to_video/image_to_video.py @@ -26,6 +26,7 @@ import numpy as np import PIL.Image import torch +from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.entrypoints.omni import Omni from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.outputs import OmniRequestOutput @@ -85,6 +86,18 @@ def parse_args() -> argparse.Namespace: default=1, help="Number of ready layers (blocks) to keep on GPU during generation.", ) + parser.add_argument( + "--cfg_parallel_size", + type=int, + default=1, + choices=[1, 2], + help="Number of GPUs used for classifier free guidance parallel size.", + ) + parser.add_argument( + "--enforce_eager", + action="store_true", + help="Disable torch.compile and force eager execution.", + ) return parser.parse_args() @@ -120,7 +133,9 @@ def main(): # Check if profiling is requested via environment variable profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR")) - + parallel_config = DiffusionParallelConfig( + cfg_parallel_size=args.cfg_parallel_size, + ) omni = Omni( model=args.model, enable_layerwise_offload=args.enable_layerwise_offload, @@ -130,12 +145,24 @@ def main(): boundary_ratio=args.boundary_ratio, flow_shift=args.flow_shift, enable_cpu_offload=args.enable_cpu_offload, + parallel_config=parallel_config, + enforce_eager=args.enforce_eager, ) if profiler_enabled: print("[Profiler] Starting profiling...") omni.start_profile() + # Print generation configuration + print(f"\n{'=' * 60}") + print("Generation Configuration:") + print(f" Model: {args.model}") + print(f" Inference steps: {args.num_inference_steps}") + print(f" Frames: {args.num_frames}") + print(f" Parallel configuration: cfg_parallel_size={args.cfg_parallel_size}") + print(f" Video size: {args.width}x{args.height}") + print(f"{'=' * 60}\n") + # omni.generate() returns Generator[OmniRequestOutput, None, None] frames = omni.generate( { diff --git a/examples/offline_inference/text_to_image/README.md b/examples/offline_inference/text_to_image/README.md index ab28b115c..9c57a621c 100644 --- a/examples/offline_inference/text_to_image/README.md +++ b/examples/offline_inference/text_to_image/README.md @@ -98,6 +98,7 @@ Key arguments: - `--output`: path to save the generated PNG. - `--vae_use_slicing`: enable VAE slicing for memory optimization. - `--vae_use_tiling`: enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). - `--enable-cpu-offload`: enable CPU offloading for diffusion models. > ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. diff --git a/examples/offline_inference/text_to_video/text_to_video.md b/examples/offline_inference/text_to_video/text_to_video.md index b5d0b2adc..04f1a2653 100644 --- a/examples/offline_inference/text_to_video/text_to_video.md +++ b/examples/offline_inference/text_to_video/text_to_video.md @@ -31,6 +31,7 @@ Key arguments: - `--output`: path to save the generated video. - `--vae_use_slicing`: enable VAE slicing for memory optimization. - `--vae_use_tiling`: enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). - `--enable-cpu-offload`: enable CPU offloading for diffusion models. > ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. diff --git a/examples/offline_inference/text_to_video/text_to_video.py b/examples/offline_inference/text_to_video/text_to_video.py index 63474987f..e9dd2d085 100644 --- a/examples/offline_inference/text_to_video/text_to_video.py +++ b/examples/offline_inference/text_to_video/text_to_video.py @@ -102,6 +102,14 @@ def parse_args() -> argparse.Namespace: default=1, help="Number of GPUs used for ring sequence parallelism.", ) + parser.add_argument( + "--cfg_parallel_size", + type=int, + default=1, + choices=[1, 2], + help="Number of GPUs used for classifier free guidance parallel size.", + ) + return parser.parse_args() @@ -132,6 +140,7 @@ def main(): parallel_config = DiffusionParallelConfig( ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree, + cfg_parallel_size=args.cfg_parallel_size, ) # Check if profiling is requested via environment variable @@ -163,7 +172,9 @@ def main(): print(f" Model: {args.model}") print(f" Inference steps: {args.num_inference_steps}") print(f" Frames: {args.num_frames}") - print(f" Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}") + print( + f" Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}" + ) print(f" Video size: {args.width}x{args.height}") print(f"{'=' * 60}\n") diff --git a/tests/diffusion/distributed/test_cfg_parallel.py b/tests/diffusion/distributed/test_cfg_parallel.py new file mode 100644 index 000000000..24e4559de --- /dev/null +++ b/tests/diffusion/distributed/test_cfg_parallel.py @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for CFG (Classifier-Free Guidance) parallel functionality. + +This test verifies that predict_noise_maybe_with_cfg produces numerically +equivalent results with and without CFG parallel using fixed random inputs. +""" + +import os + +import pytest +import torch + +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin +from vllm_omni.diffusion.distributed.parallel_state import ( + destroy_distributed_env, + get_classifier_free_guidance_rank, + get_classifier_free_guidance_world_size, + init_distributed_environment, + initialize_model_parallel, +) +from vllm_omni.platforms import current_omni_platform + + +def update_environment_variables(envs_dict: dict[str, str]): + """Update multiple environment variables.""" + for k, v in envs_dict.items(): + os.environ[k] = v + + +class SimpleTransformer(torch.nn.Module): + """Simple transformer model for testing with random initialization. + + Contains: + - Input projection (conv to hidden_dim) + - QKV projection layers + - Self-attention layer + - Output projection + """ + + def __init__(self, in_channels: int = 4, hidden_dim: int = 128, num_heads: int = 8): + super().__init__() + self.hidden_dim = hidden_dim + self.num_heads = num_heads + self.head_dim = hidden_dim // num_heads + + assert hidden_dim % num_heads == 0, "hidden_dim must be divisible by num_heads" + + # Input projection: (B, C, H, W) -> (B, hidden_dim, H, W) + self.input_proj = torch.nn.Conv2d(in_channels, hidden_dim, 1) + + # QKV projection layers + self.q_proj = torch.nn.Linear(hidden_dim, hidden_dim) + self.k_proj = torch.nn.Linear(hidden_dim, hidden_dim) + self.v_proj = torch.nn.Linear(hidden_dim, hidden_dim) + + # Output projection after attention + self.out_proj = torch.nn.Linear(hidden_dim, hidden_dim) + + # Final output projection: (B, hidden_dim, H, W) -> (B, C, H, W) + self.final_proj = torch.nn.Conv2d(hidden_dim, in_channels, 1) + + # Layer norm + self.norm1 = torch.nn.LayerNorm(hidden_dim) + self.norm2 = torch.nn.LayerNorm(hidden_dim) + + def forward(self, x: torch.Tensor, **kwargs) -> tuple[torch.Tensor]: + """Forward pass with self-attention. + + Args: + x: Input tensor of shape (B, C, H, W) + + Returns: + Output tensor of shape (B, C, H, W) + """ + B, C, H, W = x.shape + + # Input projection + x = self.input_proj(x) # (B, hidden_dim, H, W) + + # Reshape to sequence: (B, hidden_dim, H, W) -> (B, H*W, hidden_dim) + x = x.flatten(2).transpose(1, 2) # (B, H*W, hidden_dim) + + # Self-attention with residual connection + residual = x + x = self.norm1(x) + + # QKV projection + q = self.q_proj(x) # (B, H*W, hidden_dim) + k = self.k_proj(x) # (B, H*W, hidden_dim) + v = self.v_proj(x) # (B, H*W, hidden_dim) + + # Reshape for multi-head attention: (B, H*W, hidden_dim) -> (B, num_heads, H*W, head_dim) + seq_len = H * W + q = q.view(B, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + k = k.view(B, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + v = v.view(B, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + + # Scaled dot-product attention + scale = self.head_dim**-0.5 + attn_scores = torch.matmul(q, k.transpose(-2, -1)) * scale # (B, num_heads, H*W, H*W) + attn_weights = torch.nn.functional.softmax(attn_scores, dim=-1) + attn_output = torch.matmul(attn_weights, v) + attn_output = attn_output.transpose(1, 2).contiguous().view(B, seq_len, self.hidden_dim) + + attn_output = self.out_proj(attn_output) + + x = residual + attn_output + residual = x + x = self.norm2(x) + x = residual + x + x = x.transpose(1, 2).view(B, self.hidden_dim, H, W) + + out = self.final_proj(x) + + return (out,) + + +class TestCFGPipeline(CFGParallelMixin): + """Test pipeline using CFGParallelMixin.""" + + def __init__(self, in_channels: int = 4, hidden_dim: int = 128, seed: int = 42): + # Set seed BEFORE creating transformer to ensure consistent layer initialization + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + + self.transformer = SimpleTransformer(in_channels, hidden_dim) + + # Re-initialize all parameters with fixed seed for full reproducibility + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + for param in self.transformer.parameters(): + torch.nn.init.normal_(param, mean=0.0, std=0.02) + + +def _test_cfg_parallel_worker( + local_rank: int, + world_size: int, + cfg_parallel_size: int, + dtype: torch.dtype, + test_config: dict, + result_queue: torch.multiprocessing.Queue, +): + """Worker function for CFG parallel test.""" + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) + + update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": "29502", + } + ) + + init_distributed_environment() + initialize_model_parallel(cfg_parallel_size=cfg_parallel_size) + + cfg_rank = get_classifier_free_guidance_rank() + cfg_world_size = get_classifier_free_guidance_world_size() + + assert cfg_world_size == cfg_parallel_size + + # Create pipeline with same seed to ensure identical model weights across all ranks + # Note: model_seed is set inside TestCFGPipeline.__init__ + pipeline = TestCFGPipeline( + in_channels=test_config["channels"], + hidden_dim=test_config["hidden_dim"], + seed=test_config["model_seed"], + ) + pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) + pipeline.transformer.eval() # Set to eval mode for deterministic behavior + + # Create fixed inputs with explicit seed setting for reproducibility + # Set both CPU and CUDA seeds to ensure identical inputs across all ranks + torch.manual_seed(test_config["input_seed"]) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(test_config["input_seed"]) + + batch_size = test_config["batch_size"] + channels = test_config["channels"] + height = test_config["height"] + width = test_config["width"] + + # Positive input + positive_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) + + # Negative input with different seed + torch.manual_seed(test_config["input_seed"] + 1) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(test_config["input_seed"] + 1) + negative_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) + + # Prepare kwargs for predict_noise_maybe_with_cfg + positive_kwargs = {"x": positive_input} + negative_kwargs = {"x": negative_input} + + with torch.no_grad(): + # Call predict_noise_maybe_with_cfg + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=True, + true_cfg_scale=test_config["cfg_scale"], + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=test_config["cfg_normalize"], + ) + + # Only rank 0 has valid output in CFG parallel mode + if cfg_rank == 0: + assert noise_pred is not None + result_queue.put(noise_pred.cpu()) + else: + assert noise_pred is None + + destroy_distributed_env() + + +def _test_cfg_sequential_worker( + local_rank: int, + world_size: int, + dtype: torch.dtype, + test_config: dict, + result_queue: torch.multiprocessing.Queue, +): + """Worker function for sequential CFG test (baseline).""" + device = torch.device(f"{current_omni_platform.device_type}:{local_rank}") + current_omni_platform.set_device(device) + + update_environment_variables( + { + "RANK": str(local_rank), + "LOCAL_RANK": str(local_rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": "29503", + } + ) + + init_distributed_environment() + initialize_model_parallel(cfg_parallel_size=1) # No CFG parallel + + cfg_world_size = get_classifier_free_guidance_world_size() + assert cfg_world_size == 1 + + # Create pipeline with same seed to ensure identical model weights as CFG parallel + # Note: model_seed is set inside TestCFGPipeline.__init__ + pipeline = TestCFGPipeline( + in_channels=test_config["channels"], + hidden_dim=test_config["hidden_dim"], + seed=test_config["model_seed"], + ) + pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) + pipeline.transformer.eval() + + # Create fixed inputs (same seed as CFG parallel to ensure identical inputs) + # Set both CPU and CUDA seeds for full reproducibility + torch.manual_seed(test_config["input_seed"]) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(test_config["input_seed"]) + + batch_size = test_config["batch_size"] + channels = test_config["channels"] + height = test_config["height"] + width = test_config["width"] + + # Positive input + positive_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) + + # Negative input with different seed + torch.manual_seed(test_config["input_seed"] + 1) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(test_config["input_seed"] + 1) + negative_input = torch.randn(batch_size, channels, height, width, dtype=dtype, device=device) + + positive_kwargs = {"x": positive_input} + negative_kwargs = {"x": negative_input} + + with torch.no_grad(): + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=True, + true_cfg_scale=test_config["cfg_scale"], + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=test_config["cfg_normalize"], + ) + + # Sequential CFG always returns output + assert noise_pred is not None + result_queue.put(noise_pred.cpu()) + + destroy_distributed_env() + + +@pytest.mark.parametrize("cfg_parallel_size", [2]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("batch_size", [2]) +@pytest.mark.parametrize("cfg_normalize", [False, True]) +def test_predict_noise_maybe_with_cfg(cfg_parallel_size: int, dtype: torch.dtype, batch_size: int, cfg_normalize: bool): + """ + Test that predict_noise_maybe_with_cfg produces identical results + with and without CFG parallel. + + Args: + cfg_parallel_size: Number of GPUs for CFG parallel + dtype: Data type for computation + batch_size: Batch size for testing + cfg_normalize: Whether to normalize CFG output + """ + available_gpus = current_omni_platform.get_device_count() + if available_gpus < cfg_parallel_size: + pytest.skip(f"Test requires {cfg_parallel_size} GPUs but only {available_gpus} available") + + test_config = { + "batch_size": batch_size, + "channels": 4, + "height": 16, + "width": 16, + "hidden_dim": 128, + "cfg_scale": 7.5, + "cfg_normalize": cfg_normalize, + "model_seed": 42, # Fixed seed for model initialization + "input_seed": 123, # Fixed seed for input generation + } + + mp_context = torch.multiprocessing.get_context("spawn") + + manager = mp_context.Manager() + baseline_queue = manager.Queue() + cfg_parallel_queue = manager.Queue() + + # Run baseline (sequential CFG) on single GPU + torch.multiprocessing.spawn( + _test_cfg_sequential_worker, + args=(1, dtype, test_config, baseline_queue), + nprocs=1, + ) + + # Run CFG parallel on multiple GPUs + torch.multiprocessing.spawn( + _test_cfg_parallel_worker, + args=(cfg_parallel_size, cfg_parallel_size, dtype, test_config, cfg_parallel_queue), + nprocs=cfg_parallel_size, + ) + + # Get results from queues + baseline_output = baseline_queue.get() + cfg_parallel_output = cfg_parallel_queue.get() + + # Verify shapes match + assert baseline_output.shape == cfg_parallel_output.shape, ( + f"Shape mismatch: baseline {baseline_output.shape} vs CFG parallel {cfg_parallel_output.shape}" + ) + + # Verify numerical equivalence with appropriate tolerances + if dtype == torch.float32: + rtol, atol = 1e-5, 1e-5 + elif dtype == torch.bfloat16: + rtol, atol = 1e-2, 1e-2 + else: + rtol, atol = 1e-3, 1e-3 + + torch.testing.assert_close( + cfg_parallel_output, + baseline_output, + rtol=rtol, + atol=atol, + msg=( + f"CFG parallel output differs from sequential CFG\n" + f" dtype={dtype}, batch_size={batch_size}, cfg_normalize={cfg_normalize}\n" + f" Max diff: {(cfg_parallel_output - baseline_output).abs().max().item():.6e}" + ), + ) + + print( + f"✓ Test passed: cfg_size={cfg_parallel_size}, dtype={dtype}, " + f"batch_size={batch_size}, cfg_normalize={cfg_normalize}" + ) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +def test_predict_noise_without_cfg(dtype: torch.dtype): + """ + Test predict_noise_maybe_with_cfg when do_true_cfg=False. + + When CFG is disabled, only the positive branch should be computed. + This test runs on a single GPU without distributed environment. + """ + available_gpus = current_omni_platform.get_device_count() + if available_gpus < 1: + pytest.skip("Test requires at least 1 GPU") + + device = torch.device(f"{current_omni_platform.device_type}:0") + current_omni_platform.set_device(device) + + # Create pipeline without distributed environment + pipeline = TestCFGPipeline(in_channels=4, hidden_dim=128, seed=42) + pipeline.transformer = pipeline.transformer.to(device=device, dtype=dtype) + pipeline.transformer.eval() + + # Set seed for input generation + torch.manual_seed(123) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(123) + positive_input = torch.randn(1, 4, 16, 16, dtype=dtype, device=device) + + with torch.no_grad(): + noise_pred = pipeline.predict_noise_maybe_with_cfg( + do_true_cfg=False, # No CFG + true_cfg_scale=7.5, + positive_kwargs={"x": positive_input}, + negative_kwargs=None, + cfg_normalize=False, + ) + + # Should always return output when do_true_cfg=False + assert noise_pred is not None + assert noise_pred.shape == (1, 4, 16, 16) + + print(f"✓ Test passed: predict_noise without CFG (dtype={dtype})") diff --git a/vllm_omni/diffusion/data.py b/vllm_omni/diffusion/data.py index 4248071f0..f884fb6f1 100644 --- a/vllm_omni/diffusion/data.py +++ b/vllm_omni/diffusion/data.py @@ -55,6 +55,7 @@ class DiffusionParallelConfig: assert self.ulysses_degree > 0, "Ulysses degree must be > 0" assert self.ring_degree > 0, "Ring degree must be > 0" assert self.cfg_parallel_size > 0, "CFG parallel size must be > 0" + assert self.cfg_parallel_size in [1, 2], f"CFG parallel size must be 1 or 2, but got {self.cfg_parallel_size}" assert self.sequence_parallel_size == self.ulysses_degree * self.ring_degree, ( "Sequence parallel size must be equal to the product of ulysses degree and ring degree," f" but got {self.sequence_parallel_size} != {self.ulysses_degree} * {self.ring_degree}" diff --git a/vllm_omni/diffusion/distributed/cfg_parallel.py b/vllm_omni/diffusion/distributed/cfg_parallel.py new file mode 100644 index 000000000..9f86bce22 --- /dev/null +++ b/vllm_omni/diffusion/distributed/cfg_parallel.py @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Base pipeline class for Diffusion models with shared CFG functionality. +""" + +from abc import ABCMeta +from typing import Any + +import torch + +from vllm_omni.diffusion.distributed.parallel_state import ( + get_cfg_group, + get_classifier_free_guidance_rank, + get_classifier_free_guidance_world_size, +) + + +class CFGParallelMixin(metaclass=ABCMeta): + """ + Base Mixin class for Diffusion pipelines providing shared CFG methods. + + All pipelines should inherit from this class to reuse + classifier-free guidance logic. + """ + + def predict_noise_maybe_with_cfg( + self, + do_true_cfg: bool, + true_cfg_scale: float, + positive_kwargs: dict[str, Any], + negative_kwargs: dict[str, Any] | None, + cfg_normalize: bool = True, + output_slice: int | None = None, + ) -> torch.Tensor | None: + """ + Predict noise with optional classifier-free guidance. + + Args: + do_true_cfg: Whether to apply CFG + true_cfg_scale: CFG scale factor + positive_kwargs: Kwargs for positive/conditional prediction + negative_kwargs: Kwargs for negative/unconditional prediction + cfg_normalize: Whether to normalize CFG output (default: True) + output_slice: If set, slice output to [:, :output_slice] for image editing + + Returns: + Predicted noise tensor (only valid on rank 0 in CFG parallel mode) + """ + if do_true_cfg: + # Automatically detect CFG parallel configuration + cfg_parallel_ready = get_classifier_free_guidance_world_size() > 1 + + if cfg_parallel_ready: + # Enable CFG-parallel: rank0 computes positive, rank1 computes negative. + cfg_group = get_cfg_group() + cfg_rank = get_classifier_free_guidance_rank() + + if cfg_rank == 0: + local_pred = self.predict_noise(**positive_kwargs) + else: + local_pred = self.predict_noise(**negative_kwargs) + + # Slice output for image editing pipelines (remove condition latents) + if output_slice is not None: + local_pred = local_pred[:, :output_slice] + + gathered = cfg_group.all_gather(local_pred, separate_tensors=True) + + if cfg_rank == 0: + noise_pred = gathered[0] + neg_noise_pred = gathered[1] + noise_pred = self.combine_cfg_noise(noise_pred, neg_noise_pred, true_cfg_scale, cfg_normalize) + return noise_pred + else: + return None + else: + # Sequential CFG: compute both positive and negative + positive_noise_pred = self.predict_noise(**positive_kwargs) + negative_noise_pred = self.predict_noise(**negative_kwargs) + + # Slice output for image editing pipelines + if output_slice is not None: + positive_noise_pred = positive_noise_pred[:, :output_slice] + negative_noise_pred = negative_noise_pred[:, :output_slice] + + noise_pred = self.combine_cfg_noise( + positive_noise_pred, negative_noise_pred, true_cfg_scale, cfg_normalize + ) + return noise_pred + else: + # No CFG: only compute positive/conditional prediction + pred = self.predict_noise(**positive_kwargs) + if output_slice is not None: + pred = pred[:, :output_slice] + return pred + + def cfg_normalize_function(self, noise_pred: torch.Tensor, comb_pred: torch.Tensor) -> torch.Tensor: + """ + Normalize the combined noise prediction. + + Args: + noise_pred: positive noise prediction + comb_pred: combined noise prediction after CFG + + Returns: + Normalized noise prediction tensor + """ + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) + noise_pred = comb_pred * (cond_norm / noise_norm) + return noise_pred + + def combine_cfg_noise( + self, noise_pred: torch.Tensor, neg_noise_pred: torch.Tensor, true_cfg_scale: float, cfg_normalize: bool = False + ) -> torch.Tensor: + """ + Combine conditional and unconditional noise predictions with CFG. + + Args: + noise_pred: Conditional noise prediction + neg_noise_pred: Unconditional noise prediction + true_cfg_scale: CFG scale factor + cfg_normalize: Whether to normalize the combined prediction (default: False) + + Returns: + Combined noise prediction tensor + """ + comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + if cfg_normalize: + noise_pred = self.cfg_normalize_function(noise_pred, comb_pred) + else: + noise_pred = comb_pred + + return noise_pred + + def predict_noise(self, *args: Any, **kwargs: Any) -> torch.Tensor: + """ + Forward pass through transformer to predict noise. + + Subclasses should override this if they need custom behavior, + but the default implementation calls self.transformer. + """ + return self.transformer(*args, **kwargs)[0] + + def diffuse( + self, + *args: Any, + **kwargs: Any, + ) -> Any: + """ + Diffusion loop with optional classifier-free guidance. + + Subclasses MUST implement this method to define the complete + diffusion/denoising loop for their specific model. + + Typical implementation pattern: + ```python + def diffuse(self, latents, timesteps, prompt_embeds, negative_embeds, ...): + for t in timesteps: + # Prepare kwargs for positive and negative predictions + positive_kwargs = {...} + negative_kwargs = {...} + + # Predict noise with automatic CFG handling + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=True, + true_cfg_scale=self.guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + ) + + # Step scheduler with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg( + noise_pred, t, latents, do_true_cfg=True + ) + + return latents + ``` + """ + raise NotImplementedError("Subclasses must implement diffuse") + + def scheduler_step(self, noise_pred: torch.Tensor, t: torch.Tensor, latents: torch.Tensor) -> torch.Tensor: + """ + Step the scheduler. + + Args: + noise_pred: Predicted noise + t: Current timestep + latents: Current latents + + Returns: + Updated latents after scheduler step + """ + return self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + def scheduler_step_maybe_with_cfg( + self, noise_pred: torch.Tensor, t: torch.Tensor, latents: torch.Tensor, do_true_cfg: bool + ) -> torch.Tensor: + """ + Step the scheduler with (maybe) automatic CFG parallel synchronization. + + In CFG parallel mode, only rank 0 computes the scheduler step, + then broadcasts the result to other ranks. + + Args: + noise_pred: Predicted noise (only valid on rank 0 in CFG parallel) + t: Current timestep + latents: Current latents + do_true_cfg: Whether CFG is enabled + + Returns: + Updated latents (synchronized across all CFG ranks) + """ + # Automatically detect CFG parallel configuration + cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1 + + if cfg_parallel_ready: + cfg_group = get_cfg_group() + cfg_rank = get_classifier_free_guidance_rank() + + # Only rank 0 computes the scheduler step + if cfg_rank == 0: + latents = self.scheduler_step(noise_pred, t, latents) + + # Broadcast the updated latents to all ranks + latents = latents.contiguous() + cfg_group.broadcast(latents, src=0) + else: + # No CFG parallel: directly compute scheduler step + latents = self.scheduler_step(noise_pred, t, latents) + + return latents diff --git a/vllm_omni/diffusion/models/flux/pipeline_flux.py b/vllm_omni/diffusion/models/flux/pipeline_flux.py index a91601b7e..b90aaa8ca 100644 --- a/vllm_omni/diffusion/models/flux/pipeline_flux.py +++ b/vllm_omni/diffusion/models/flux/pipeline_flux.py @@ -22,6 +22,7 @@ from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5Tokeniz from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.flux import FluxTransformer2DModel @@ -554,6 +555,21 @@ class FluxPipeline( latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] return latents + def check_cfg_parallel_validity(self, true_cfg_scale: float, has_neg_prompt: bool): + if get_classifier_free_guidance_world_size() == 1: + return True + + if true_cfg_scale <= 1: + logger.warning("CFG parallel is NOT working correctly when true_cfg_scale <= 1.") + return False + + if not has_neg_prompt: + logger.warning( + "CFG parallel is NOT working correctly when there is no negative prompt or negative prompt embeddings." + ) + return False + return True + def forward( self, req: OmniDiffusionRequest, @@ -637,6 +653,8 @@ class FluxPipeline( ) do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt) + ( prompt_embeds, pooled_prompt_embeds, diff --git a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py index 2eac7b2ec..e1ef706c3 100644 --- a/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py +++ b/vllm_omni/diffusion/models/flux2_klein/pipeline_flux2_klein.py @@ -36,6 +36,7 @@ from vllm.logger import init_logger from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.flux2_klein.flux2_klein_transformer import ( @@ -178,7 +179,7 @@ def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: return float(mu) -class Flux2KleinPipeline(nn.Module, SupportImageInput): +class Flux2KleinPipeline(nn.Module, CFGParallelMixin, SupportImageInput): """Flux2 klein pipeline for text-to-image generation.""" support_image_input = True @@ -923,38 +924,44 @@ class Flux2KleinPipeline(nn.Module, SupportImageInput): latent_model_input = torch.cat([latents, image_latents], dim=1).to(self.transformer.dtype) latent_image_ids = torch.cat([latent_ids, image_latent_ids], dim=1) - noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=None, - encoder_hidden_states=prompt_embeds, - txt_ids=text_ids, - img_ids=latent_image_ids, - joint_attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - - noise_pred = noise_pred[:, : latents.size(1) :] - + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "guidance": None, + "encoder_hidden_states": prompt_embeds, + "txt_ids": text_ids, + "img_ids": latent_image_ids, + "joint_attention_kwargs": self.attention_kwargs, + "return_dict": False, + } if self.do_classifier_free_guidance: - neg_noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=None, - encoder_hidden_states=negative_prompt_embeds, - txt_ids=negative_text_ids, - img_ids=latent_image_ids, - joint_attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - neg_noise_pred = neg_noise_pred[:, : latents.size(1) :] - noise_pred = neg_noise_pred + guidance_scale * (noise_pred - neg_noise_pred) + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "guidance": None, + "encoder_hidden_states": negative_prompt_embeds, + "txt_ids": negative_text_ids, + "img_ids": latent_image_ids, + "joint_attention_kwargs": self.attention_kwargs, + "return_dict": False, + } + else: + negative_kwargs = None - latents_dtype = latents.dtype - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + # For editing pipelines, we need to slice the output to remove condition latents + output_slice = latents.size(1) if image_latents is not None else None - if latents.dtype != latents_dtype and torch.backends.mps.is_available(): - latents = latents.to(latents_dtype) + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=self.do_classifier_free_guidance, + true_cfg_scale=guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + output_slice=output_slice, + ) + + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, self.do_classifier_free_guidance) if callback_on_step_end is not None: callback_kwargs = {} diff --git a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py index 8b616ec45..09f409f31 100644 --- a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py +++ b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image.py @@ -8,6 +8,7 @@ import json import os import re from collections.abc import Iterable +from functools import partial from typing import Any import numpy as np @@ -23,6 +24,7 @@ from vllm.logger import init_logger from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.longcat_image.longcat_image_transformer import LongCatImageTransformer2DModel @@ -197,9 +199,7 @@ def get_prompt_language(prompt): return "en" -class LongCatImagePipeline( - nn.Module, -): +class LongCatImagePipeline(nn.Module, CFGParallelMixin): def __init__( self, *, @@ -460,6 +460,16 @@ class LongCatImagePipeline( f" {negative_prompt_embeds}. Please make sure to only forward one of the two." ) + def cfg_normalize_function(self, noise_pred, comb_pred, cfg_renorm_min=0.0): + """ + Normalize the combined noise prediction. + """ + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) + scale = (cond_norm / (noise_norm + 1e-8)).clamp(min=cfg_renorm_min, max=1.0) + noise_pred = comb_pred * scale + return noise_pred + def forward( self, req: OmniDiffusionRequest, @@ -604,6 +614,9 @@ class LongCatImagePipeline( if self.do_classifier_free_guidance: negative_prompt_embeds = negative_prompt_embeds.to(device) + # custom partial function with cfg_renorm_min + self.cfg_normalize_function = partial(self.cfg_normalize_function, cfg_renorm_min=cfg_renorm_min) + # 6. Denoising loop for i, t in enumerate(timesteps): if self._interrupt: @@ -612,43 +625,37 @@ class LongCatImagePipeline( self._current_timestep = t timestep = t.expand(latents.shape[0]).to(latents.dtype) - noise_pred_text = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states=prompt_embeds, - txt_ids=text_ids, - img_ids=latent_image_ids, - return_dict=False, - )[0] - + positive_kwargs = { + "hidden_states": latents, + "timestep": timestep / 1000, + "guidance": guidance, + "encoder_hidden_states": prompt_embeds, + "txt_ids": text_ids, + "img_ids": latent_image_ids, + "return_dict": False, + } if self.do_classifier_free_guidance: - noise_pred_uncond = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - encoder_hidden_states=negative_prompt_embeds, - txt_ids=negative_text_ids, - img_ids=latent_image_ids, - return_dict=False, - )[0] - noise_pred = noise_pred_uncond + self._guidance_scale * (noise_pred_text - noise_pred_uncond) - - if enable_cfg_renorm: - cond_norm = torch.norm(noise_pred_text, dim=-1, keepdim=True) - noise_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - scale = (cond_norm / (noise_norm + 1e-8)).clamp(min=cfg_renorm_min, max=1.0) - noise_pred = noise_pred * scale + negative_kwargs = { + "hidden_states": latents, + "timestep": timestep / 1000, + "encoder_hidden_states": negative_prompt_embeds, + "txt_ids": negative_text_ids, + "img_ids": latent_image_ids, + "return_dict": False, + } else: - noise_pred = noise_pred_text + negative_kwargs = None - # compute the previous noisy sample x_t -> x_t-1 - latents_dtype = latents.dtype - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=self.do_classifier_free_guidance, + true_cfg_scale=guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=enable_cfg_renorm, + ) - if latents.dtype != latents_dtype: - if torch.backends.mps.is_available(): - # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 - latents = latents.to(latents_dtype) + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, self.do_classifier_free_guidance) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py index f2c3fd648..a34a2cca3 100644 --- a/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py +++ b/vllm_omni/diffusion/models/longcat_image/pipeline_longcat_image_edit.py @@ -26,6 +26,7 @@ from vllm.logger import init_logger from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportImageInput @@ -215,7 +216,7 @@ def split_quotation(prompt, quote_pairs=None): return result -class LongCatImageEditPipeline(nn.Module, SupportImageInput): +class LongCatImageEditPipeline(nn.Module, CFGParallelMixin, SupportImageInput): def __init__( self, *, @@ -652,38 +653,39 @@ class LongCatImageEditPipeline(nn.Module, SupportImageInput): latent_model_input = torch.cat([latents, image_latents], dim=1) timestep = t.expand(latent_model_input.shape[0]).to(latents.dtype) + do_true_cfg = guidance_scale > 1 + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "guidance": guidance, + "encoder_hidden_states": prompt_embeds, + "txt_ids": text_ids, + "img_ids": latent_image_ids, + "return_dict": False, + } - noise_pred_text = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states=prompt_embeds, - txt_ids=text_ids, - img_ids=latent_image_ids, - return_dict=False, - )[0] - noise_pred_text = noise_pred_text[:, :image_seq_len] - if guidance_scale > 1: - noise_pred_uncond = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - encoder_hidden_states=negative_prompt_embeds, - txt_ids=negative_text_ids, - img_ids=latent_image_ids, - return_dict=False, - )[0] - noise_pred_uncond = noise_pred_uncond[:, :image_seq_len] - noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "encoder_hidden_states": negative_prompt_embeds, + "txt_ids": negative_text_ids, + "img_ids": latent_image_ids, + "return_dict": False, + } else: - noise_pred = noise_pred_text - # compute the previous noisy sample x_t -> x_t-1 - latents_dtype = latents.dtype - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + negative_kwargs = None - if latents.dtype != latents_dtype: - if torch.backends.mps.is_available(): - # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 - latents = latents.to(latents_dtype) + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + output_slice=image_seq_len, + ) + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py b/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py index a87410a6c..963f1c483 100644 --- a/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py +++ b/vllm_omni/diffusion/models/ovis_image/pipeline_ovis_image.py @@ -23,18 +23,19 @@ from typing import Any import numpy as np import torch -import torch.nn as nn from diffusers.image_processor import VaeImageProcessor from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL from diffusers.schedulers.scheduling_flow_match_euler_discrete import ( FlowMatchEulerDiscreteScheduler, ) from diffusers.utils.torch_utils import randn_tensor +from torch import nn from transformers import Qwen2TokenizerFast, Qwen3Model from vllm.logger import init_logger from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.ovis_image.ovis_image_transformer import OvisImageTransformer2DModel @@ -139,9 +140,7 @@ def retrieve_timesteps( return timesteps, num_inference_steps -class OvisImagePipeline( - nn.Module, -): +class OvisImagePipeline(nn.Module, CFGParallelMixin): def __init__( self, *, @@ -431,68 +430,77 @@ class OvisImagePipeline( ) return timesteps, num_inference_steps - def denoising( + def diffuse( self, - latents, - timesteps, - prompt_embeds, - negative_prompt_embeds, - guidance_scale, - do_classifier_free_guidance, - callback_on_step_end, - callback_on_step_end_tensor_inputs, - text_ids, - negative_text_ids, - latent_image_ids, - ): + latents: torch.Tensor, + timesteps: torch.Tensor, + prompt_embeds: torch.Tensor, + negative_prompt_embeds: torch.Tensor, + text_ids: torch.Tensor, + negative_text_ids: torch.Tensor, + latent_image_ids: torch.Tensor, + do_true_cfg: bool, + guidance_scale: float, + cfg_normalize: bool = False, + ) -> torch.Tensor: + """ + Diffusion loop with optional classifier-free guidance. + + Args: + latents: Noise latents to denoise + timesteps: Diffusion timesteps + prompt_embeds: Positive prompt embeddings + negative_prompt_embeds: Negative prompt embeddings + text_ids: Position IDs for positive text + negative_text_ids: Position IDs for negative text + latent_image_ids: Position IDs for image latents + do_true_cfg: Whether to apply CFG + guidance_scale: CFG scale factor + cfg_normalize: Whether to normalize CFG output (default: False) + + Returns: + Denoised latents + """ self.scheduler.set_begin_index(0) + for i, t in enumerate(timesteps): - if self._interrupt: + if self.interrupt: break self._current_timestep = t - # broadcast to batch dimension in a way that's compatible with ONNX/Core ML timestep = t.expand(latents.shape[0]).to(latents.dtype) - noise_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - encoder_hidden_states=prompt_embeds, - txt_ids=text_ids, - img_ids=latent_image_ids, - return_dict=False, - )[0] + positive_kwargs = { + "hidden_states": latents, + "timestep": timestep / 1000, + "encoder_hidden_states": prompt_embeds, + "txt_ids": text_ids, + "img_ids": latent_image_ids, + "return_dict": False, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latents, + "timestep": timestep / 1000, + "encoder_hidden_states": negative_prompt_embeds, + "txt_ids": negative_text_ids, + "img_ids": latent_image_ids, + "return_dict": False, + } + else: + negative_kwargs = None - if do_classifier_free_guidance: - neg_noise_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - encoder_hidden_states=negative_prompt_embeds, - txt_ids=negative_text_ids, - img_ids=latent_image_ids, - return_dict=False, - )[0] + # Predict noise with automatic CFG parallel handling + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg, + guidance_scale, + positive_kwargs, + negative_kwargs, + cfg_normalize, + ) - noise_pred = neg_noise_pred + guidance_scale * (noise_pred - neg_noise_pred) - - # compute the previous noisy sample x_t -> x_t-1 - latents_dtype = latents.dtype - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - - if latents.dtype != latents_dtype: - if torch.backends.mps.is_available(): - # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 - latents = latents.to(latents_dtype) - - # Not used in this pipeline - # if callback_on_step_end is not None: - # callback_kwargs = {} - # for k in callback_on_step_end_tensor_inputs: - # callback_kwargs[k] = locals()[k] - # callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) - - # latents = callback_outputs.pop("latents", latents) - # prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) return latents @@ -704,23 +712,18 @@ class OvisImagePipeline( if self.joint_attention_kwargs is None: self._joint_attention_kwargs = {} - # 6. Denoising loop - - # We set the index here to remove DtoH sync, helpful especially during compilation - # Check out more details here: https://github.com/huggingface/diffusers/pull/11696 - - latents = self.denoising( - latents, - timesteps, - prompt_embeds, - negative_prompt_embeds, - guidance_scale, - do_classifier_free_guidance, - callback_on_step_end, - callback_on_step_end_tensor_inputs, - text_ids, - negative_text_ids, - latent_image_ids, + # 6. Denoising loop using diffuse method + latents = self.diffuse( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds if do_classifier_free_guidance else None, + text_ids=text_ids, + negative_text_ids=negative_text_ids if do_classifier_free_guidance else None, + latent_image_ids=latent_image_ids, + do_true_cfg=do_classifier_free_guidance, + guidance_scale=guidance_scale, + cfg_normalize=False, ) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/qwen_image/__init__.py b/vllm_omni/diffusion/models/qwen_image/__init__.py index 84fa2259d..4b823ec75 100644 --- a/vllm_omni/diffusion/models/qwen_image/__init__.py +++ b/vllm_omni/diffusion/models/qwen_image/__init__.py @@ -2,6 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Qwen Image diffusion model components.""" +from vllm_omni.diffusion.models.qwen_image.cfg_parallel import ( + QwenImageCFGParallelMixin, +) from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import ( QwenImagePipeline, get_qwen_image_post_process_func, @@ -11,6 +14,7 @@ from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( ) __all__ = [ + "QwenImageCFGParallelMixin", "QwenImagePipeline", "QwenImageTransformer2DModel", "get_qwen_image_post_process_func", diff --git a/vllm_omni/diffusion/models/qwen_image/cfg_parallel.py b/vllm_omni/diffusion/models/qwen_image/cfg_parallel.py new file mode 100644 index 000000000..9a882f7bf --- /dev/null +++ b/vllm_omni/diffusion/models/qwen_image/cfg_parallel.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CFG Parallel Mixin for Qwen Image series +Shared by +- QwenImagePipeline +- QwenImageEditPipeline +- QwenImageEditPlusPipeline +- QwenImageLayeredPipeline +""" + +import logging +from typing import Any + +import torch + +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin +from vllm_omni.diffusion.distributed.parallel_state import get_classifier_free_guidance_world_size + +logger = logging.getLogger(__name__) + + +class QwenImageCFGParallelMixin(CFGParallelMixin): + """ + Base Mixin class for Qwen Image pipelines providing shared CFG methods. + """ + + def diffuse( + self, + prompt_embeds: torch.Tensor, + prompt_embeds_mask: torch.Tensor, + negative_prompt_embeds: torch.Tensor, + negative_prompt_embeds_mask: torch.Tensor, + latents: torch.Tensor, + img_shapes: torch.Tensor, + txt_seq_lens: torch.Tensor, + negative_txt_seq_lens: torch.Tensor, + timesteps: torch.Tensor, + do_true_cfg: bool, + guidance: torch.Tensor, + true_cfg_scale: float, + image_latents: torch.Tensor | None = None, + cfg_normalize: bool = True, + additional_transformer_kwargs: dict[str, Any] | None = None, + ) -> torch.Tensor: + """ + Diffusion loop with optional classifier-free guidance. + + Args: + prompt_embeds: Positive prompt embeddings + prompt_embeds_mask: Mask for positive prompt + negative_prompt_embeds: Negative prompt embeddings + negative_prompt_embeds_mask: Mask for negative prompt + latents: Noise latents to denoise + img_shapes: Image shape information + txt_seq_lens: Text sequence lengths for positive prompts + negative_txt_seq_lens: Text sequence lengths for negative prompts + timesteps: Diffusion timesteps + do_true_cfg: Whether to apply CFG + guidance: Guidance scale tensor + true_cfg_scale: CFG scale factor + image_latents: Conditional image latents for editing (default: None) + cfg_normalize: Whether to normalize CFG output (default: True) + additional_transformer_kwargs: Extra kwargs to pass to transformer (default: None) + + Returns: + Denoised latents + """ + self.scheduler.set_begin_index(0) + self.transformer.do_true_cfg = do_true_cfg + additional_transformer_kwargs = additional_transformer_kwargs or {} + + for i, t in enumerate(timesteps): + if self.interrupt: + continue + self._current_timestep = t + + # Broadcast timestep to match batch size + timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) + + # Concatenate image latents with noise latents if available (for editing pipelines) + latent_model_input = latents + if image_latents is not None: + latent_model_input = torch.cat([latents, image_latents], dim=1) + + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "guidance": guidance, + "encoder_hidden_states_mask": prompt_embeds_mask, + "encoder_hidden_states": prompt_embeds, + "img_shapes": img_shapes, + "txt_seq_lens": txt_seq_lens, + **additional_transformer_kwargs, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep / 1000, + "guidance": guidance, + "encoder_hidden_states_mask": negative_prompt_embeds_mask, + "encoder_hidden_states": negative_prompt_embeds, + "img_shapes": img_shapes, + "txt_seq_lens": negative_txt_seq_lens, + **additional_transformer_kwargs, + } + else: + negative_kwargs = None + + # For editing pipelines, we need to slice the output to remove condition latents + output_slice = latents.size(1) if image_latents is not None else None + + # Predict noise with automatic CFG parallel handling + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg, + true_cfg_scale, + positive_kwargs, + negative_kwargs, + cfg_normalize, + output_slice, + ) + + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + + return latents + + def check_cfg_parallel_validity(self, true_cfg_scale: float, has_neg_prompt: bool): + """ + Validate whether CFG parallel is properly configured for the current generation request. + + When CFG parallel is enabled (cfg_parallel_world_size > 1), this method verifies that the necessary + conditions are met for correct parallel execution. If validation fails, a warning is + logged to help identify configuration issues. + + Args: + true_cfg_scale: The classifier-free guidance scale value. Must be > 1 for CFG to + have an effect. + has_neg_prompt: Whether negative prompts or negative prompt embeddings are provided. + Required for CFG to perform unconditional prediction. + + Returns: + True if CFG parallel is disabled or all validation checks pass, False otherwise. + + Note: + When CFG parallel is disabled (world_size == 1), this method always returns True + as no parallel-specific validation is needed. + """ + if get_classifier_free_guidance_world_size() == 1: + return True + + if true_cfg_scale <= 1: + logger.warning("CFG parallel is NOT working correctly when true_cfg_scale <= 1.") + return False + + if not has_neg_prompt: + logger.warning( + "CFG parallel is NOT working correctly when there is no negative prompt or negative prompt embeddings." + ) + return False + return True diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py index e0a37b8bc..d85d98b5b 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py @@ -25,13 +25,11 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.parallel_state import ( - get_cfg_group, - get_classifier_free_guidance_rank, - get_classifier_free_guidance_world_size, -) from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader +from vllm_omni.diffusion.models.qwen_image.cfg_parallel import ( + QwenImageCFGParallelMixin, +) from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( QwenImageTransformer2DModel, ) @@ -239,9 +237,7 @@ def apply_rotary_emb_qwen( return x_out.type_as(x) -class QwenImagePipeline( - nn.Module, -): +class QwenImagePipeline(nn.Module, QwenImageCFGParallelMixin): def __init__( self, *, @@ -536,109 +532,6 @@ class QwenImagePipeline( def interrupt(self): return self._interrupt - def diffuse( - self, - prompt_embeds, - prompt_embeds_mask, - negative_prompt_embeds, - negative_prompt_embeds_mask, - latents, - img_shapes, - txt_seq_lens, - negative_txt_seq_lens, - timesteps, - do_true_cfg, - guidance, - true_cfg_scale, - ): - self.scheduler.set_begin_index(0) - for i, t in enumerate(timesteps): - if self.interrupt: - continue - self._current_timestep = t - - # Broadcast timestep to match batch size - timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) - - # Enable CFG-parallel: rank0 computes positive, rank1 computes negative. - cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1 - - self.transformer.do_true_cfg = do_true_cfg - - if cfg_parallel_ready: - cfg_group = get_cfg_group() - cfg_rank = get_classifier_free_guidance_rank() - - if cfg_rank == 0: - local_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - else: - local_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - - gathered = cfg_group.all_gather(local_pred, separate_tensors=True) - if cfg_rank == 0: - noise_pred = gathered[0] - neg_noise_pred = gathered[1] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - cfg_group.broadcast(latents, src=0) - - else: - # Forward pass for positive prompt (or unconditional if no CFG) - noise_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - # Forward pass for negative prompt (CFG) - if do_true_cfg: - neg_noise_pred = self.transformer( - hidden_states=latents, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - # compute the previous noisy sample x_t -> x_t-1 - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - return latents - def forward( self, req: OmniDiffusionRequest, @@ -720,6 +613,8 @@ class QwenImagePipeline( ) do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt) + prompt_embeds, prompt_embeds_mask = self.encode_prompt( prompt=prompt, prompt_embeds=prompt_embeds, @@ -790,6 +685,12 @@ class QwenImagePipeline( do_true_cfg, guidance, true_cfg_scale, + image_latents=None, + cfg_normalize=True, + additional_transformer_kwargs={ + "return_dict": False, + "attention_kwargs": self.attention_kwargs, + }, ) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py index 4241370ba..78fd92c9d 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py @@ -25,14 +25,12 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.parallel_state import ( - get_cfg_group, - get_classifier_free_guidance_rank, - get_classifier_free_guidance_world_size, -) from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportImageInput +from vllm_omni.diffusion.models.qwen_image.cfg_parallel import ( + QwenImageCFGParallelMixin, +) from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import calculate_shift from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( QwenImageTransformer2DModel, @@ -213,10 +211,7 @@ def retrieve_latents( raise AttributeError("Could not access latents of provided encoder_output") -class QwenImageEditPipeline( - nn.Module, - SupportImageInput, -): +class QwenImageEditPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin): def __init__( self, *, @@ -598,118 +593,6 @@ class QwenImageEditPipeline( def interrupt(self): return self._interrupt - def diffuse( - self, - prompt_embeds, - prompt_embeds_mask, - negative_prompt_embeds, - negative_prompt_embeds_mask, - latents, - image_latents, - img_shapes, - txt_seq_lens, - negative_txt_seq_lens, - timesteps, - do_true_cfg, - guidance, - true_cfg_scale, - ): - """Diffusion loop with optional image conditioning.""" - self.scheduler.set_begin_index(0) - for i, t in enumerate(timesteps): - if self.interrupt: - continue - self._current_timestep = t - # broadcast to batch dimension and place on same device/dtype as latents - timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) - - # Concatenate image latents with noise latents if available - latent_model_input = latents - if image_latents is not None: - latent_model_input = torch.cat([latents, image_latents], dim=1) - - self.transformer.do_true_cfg = do_true_cfg # used in teacache hook - cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1 - - if cfg_parallel_ready: - cfg_group = get_cfg_group() - cfg_rank = get_classifier_free_guidance_rank() - - if cfg_rank == 0: - local_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - else: - local_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - - local_pred = local_pred[:, : latents.size(1)] - - gathered = cfg_group.all_gather(local_pred, separate_tensors=True) - if cfg_rank == 0: - noise_pred = gathered[0] - neg_noise_pred = gathered[1] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - cfg_group.broadcast(latents, src=0) - - else: - # Forward pass for positive prompt (or unconditional if no CFG) - noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - noise_pred = noise_pred[:, : latents.size(1)] - - # Forward pass for negative prompt (CFG) - if do_true_cfg: - neg_noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - neg_noise_pred = neg_noise_pred[:, : latents.size(1)] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - # compute the previous noisy sample x_t -> x_t-1 - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - return latents - def forward( self, req: OmniDiffusionRequest, @@ -823,6 +706,8 @@ class QwenImageEditPipeline( ) do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt) + prompt_embeds, prompt_embeds_mask = self.encode_prompt( prompt=prompt, image=prompt_image, # Use resized image for prompt encoding @@ -886,7 +771,6 @@ class QwenImageEditPipeline( negative_prompt_embeds, negative_prompt_embeds_mask, latents, - image_latents, img_shapes, txt_seq_lens, negative_txt_seq_lens, @@ -894,6 +778,12 @@ class QwenImageEditPipeline( do_true_cfg, guidance, true_cfg_scale, + image_latents=image_latents, + cfg_normalize=True, + additional_transformer_kwargs={ + "return_dict": False, + "attention_kwargs": self.attention_kwargs, + }, ) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py index 9402c1f7c..00e775802 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit_plus.py @@ -23,14 +23,12 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.parallel_state import ( - get_cfg_group, - get_classifier_free_guidance_rank, - get_classifier_free_guidance_world_size, -) from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportImageInput +from vllm_omni.diffusion.models.qwen_image.cfg_parallel import ( + QwenImageCFGParallelMixin, +) from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image import calculate_shift from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import ( calculate_dimensions, @@ -168,7 +166,7 @@ def get_qwen_image_edit_plus_post_process_func( return post_process_func -class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): +class QwenImageEditPlusPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin): def __init__( self, *, @@ -530,116 +528,6 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): def interrupt(self): return self._interrupt - def diffuse( - self, - prompt_embeds, - prompt_embeds_mask, - negative_prompt_embeds, - negative_prompt_embeds_mask, - latents, - image_latents, - img_shapes, - txt_seq_lens, - negative_txt_seq_lens, - timesteps, - do_true_cfg, - guidance, - true_cfg_scale, - ): - """Diffusion loop with optional image conditioning.""" - self.scheduler.set_begin_index(0) - for i, t in enumerate(timesteps): - if self.interrupt: - continue - self._current_timestep = t - # broadcast to batch dimension and place on same device/dtype as latents - timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) - - # Concatenate image latents with noise latents if available - latent_model_input = latents - if image_latents is not None: - latent_model_input = torch.cat([latents, image_latents], dim=1) - - self.transformer.do_true_cfg = do_true_cfg - cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1 - - if cfg_parallel_ready: - cfg_group = get_cfg_group() - cfg_rank = get_classifier_free_guidance_rank() - - if cfg_rank == 0: - local_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - else: - local_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - - local_pred = local_pred[:, : latents.size(1)] - - gathered = cfg_group.all_gather(local_pred, separate_tensors=True) - if cfg_rank == 0: - noise_pred = gathered[0] - neg_noise_pred = gathered[1] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - cfg_group.broadcast(latents, src=0) - - else: - noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - noise_pred = noise_pred[:, : latents.size(1)] - - if do_true_cfg: - neg_noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - return_dict=False, - )[0] - neg_noise_pred = neg_noise_pred[:, : latents.size(1)] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - # compute the previous noisy sample x_t -> x_t-1 - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - return latents - def forward( self, req: OmniDiffusionRequest, @@ -773,6 +661,8 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): ) do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt) + prompt_embeds, prompt_embeds_mask = self.encode_prompt( prompt=prompt, image=condition_images, # Use condition images for prompt encoding @@ -840,7 +730,6 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): negative_prompt_embeds, negative_prompt_embeds_mask, latents, - image_latents, img_shapes, txt_seq_lens, negative_txt_seq_lens, @@ -848,6 +737,12 @@ class QwenImageEditPlusPipeline(nn.Module, SupportImageInput): do_true_cfg, guidance, true_cfg_scale, + image_latents=image_latents, + cfg_normalize=True, + additional_transformer_kwargs={ + "return_dict": False, + "attention_kwargs": self.attention_kwargs, + }, ) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py index 1001e8a14..d200764eb 100644 --- a/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py +++ b/vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_layered.py @@ -22,17 +22,15 @@ from transformers import Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwe from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig -from vllm_omni.diffusion.distributed.parallel_state import ( - get_cfg_group, - get_classifier_free_guidance_rank, - get_classifier_free_guidance_world_size, -) from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportImageInput from vllm_omni.diffusion.models.qwen_image.autoencoder_kl_qwenimage import ( AutoencoderKLQwenImage, ) +from vllm_omni.diffusion.models.qwen_image.cfg_parallel import ( + QwenImageCFGParallelMixin, +) from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import ( QwenImageTransformer2DModel, ) @@ -191,7 +189,7 @@ def retrieve_latents( raise AttributeError("Could not access latents of provided encoder_output") -class QwenImageLayeredPipeline(nn.Module, SupportImageInput): +class QwenImageLayeredPipeline(nn.Module, SupportImageInput, QwenImageCFGParallelMixin): color_format = "RGBA" def __init__( @@ -553,138 +551,6 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as return latents - def diffuse( - self, - prompt_embeds, - prompt_embeds_mask, - negative_prompt_embeds, - negative_prompt_embeds_mask, - latents, - image_latents, - img_shapes, - txt_seq_lens, - negative_txt_seq_lens, - timesteps, - do_true_cfg, - guidance, - true_cfg_scale, - cfg_normalize, - is_rgb, - ): - self.scheduler.set_begin_index(0) - for i, t in enumerate(timesteps): - if self.interrupt: - continue - - self._current_timestep = t - - latent_model_input = latents - if image_latents is not None: - latent_model_input = torch.cat([latents, image_latents], dim=1) - - # broadcast to batch dimension in a way that's compatible with ONNX/Core ML - timestep = t.expand(latents.shape[0]).to(latents.dtype) - - # Enable CFG-parallel: rank0 computes positive, rank1 computes negative. - cfg_parallel_ready = do_true_cfg and get_classifier_free_guidance_world_size() > 1 - - self.transformer.do_true_cfg = do_true_cfg # used in teacache hook - - if cfg_parallel_ready: - cfg_group = get_cfg_group() - cfg_rank = get_classifier_free_guidance_rank() - - if cfg_rank == 0: - local_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - additional_t_cond=is_rgb, - return_dict=False, - )[0] - local_pred = local_pred[:, : latents.size(1)] - else: - local_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - additional_t_cond=is_rgb, - return_dict=False, - )[0] - local_pred = local_pred[:, : latents.size(1)] - - gathered = cfg_group.all_gather(local_pred, separate_tensors=True) - if cfg_rank == 0: - noise_pred = gathered[0] - neg_noise_pred = gathered[1] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - - if cfg_normalize: - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - else: - noise_pred = comb_pred - - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - cfg_group.broadcast(latents, src=0) - - else: - # Forward pass for positive prompt (or unconditional if no CFG) - noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=prompt_embeds_mask, - encoder_hidden_states=prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=txt_seq_lens, - attention_kwargs=self.attention_kwargs, - additional_t_cond=is_rgb, - return_dict=False, - )[0] - noise_pred = noise_pred[:, : latents.size(1)] - - if do_true_cfg: - neg_noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep / 1000, - guidance=guidance, - encoder_hidden_states_mask=negative_prompt_embeds_mask, - encoder_hidden_states=negative_prompt_embeds, - img_shapes=img_shapes, - txt_seq_lens=negative_txt_seq_lens, - attention_kwargs=self.attention_kwargs, - additional_t_cond=is_rgb, - return_dict=False, - )[0] - - neg_noise_pred = neg_noise_pred[:, : latents.size(1)] - comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) - - if cfg_normalize: - cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) - noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) - noise_pred = comb_pred * (cond_norm / noise_norm) - else: - noise_pred = comb_pred - - # compute the previous noisy sample x_t -> x_t-1 - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] - - # no need callback now - return latents - @property def guidance_scale(self): return self._guidance_scale @@ -840,6 +706,8 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as ) do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + self.check_cfg_parallel_validity(true_cfg_scale, has_neg_prompt) + prompt_embeds, prompt_embeds_mask = self.encode_prompt( prompt=prompt, prompt_embeds=prompt_embeds, @@ -924,7 +792,6 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as negative_prompt_embeds, negative_prompt_embeds_mask, latents, - image_latents, img_shapes, txt_seq_lens, negative_txt_seq_lens, @@ -932,8 +799,13 @@ the image\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>as do_true_cfg, guidance, true_cfg_scale, - cfg_normalize, - is_rgb, + image_latents=image_latents, + cfg_normalize=cfg_normalize, + additional_transformer_kwargs={ + "return_dict": False, + "additional_t_cond": is_rgb, + "attention_kwargs": self.attention_kwargs, + }, ) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/sd3/pipeline_sd3.py b/vllm_omni/diffusion/models/sd3/pipeline_sd3.py index 34a0eb6c1..3668c132f 100644 --- a/vllm_omni/diffusion/models/sd3/pipeline_sd3.py +++ b/vllm_omni/diffusion/models/sd3/pipeline_sd3.py @@ -16,6 +16,7 @@ from transformers import CLIPTextModelWithProjection, CLIPTokenizer, T5EncoderMo from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.sd3.sd3_transformer import ( @@ -126,9 +127,7 @@ def retrieve_timesteps( return timesteps, num_inference_steps -class StableDiffusion3Pipeline( - nn.Module, -): +class StableDiffusion3Pipeline(nn.Module, CFGParallelMixin): def __init__( self, *, @@ -498,15 +497,35 @@ class StableDiffusion3Pipeline( def diffuse( self, - prompt_embeds, - pooled_prompt_embeds, - negative_prompt_embeds, - negative_pooled_prompt_embeds, - latents, - timesteps, - do_cfg, - ): + latents: torch.Tensor, + timesteps: torch.Tensor, + prompt_embeds: torch.Tensor, + pooled_prompt_embeds: torch.Tensor, + negative_prompt_embeds: torch.Tensor, + negative_pooled_prompt_embeds: torch.Tensor, + do_true_cfg: bool, + guidance_scale: float, + cfg_normalize: bool = False, + ) -> torch.Tensor: + """ + Diffusion loop with optional classifier-free guidance. + + Args: + latents: Noise latents to denoise + timesteps: Diffusion timesteps + prompt_embeds: Positive prompt embeddings + pooled_prompt_embeds: Pooled positive prompt embeddings + negative_prompt_embeds: Negative prompt embeddings + negative_pooled_prompt_embeds: Pooled negative prompt embeddings + do_true_cfg: Whether to apply CFG + guidance_scale: CFG scale factor + cfg_normalize: Whether to normalize CFG output (default: False) + + Returns: + Denoised latents + """ self.scheduler.set_begin_index(0) + for _, t in enumerate(timesteps): if self.interrupt: continue @@ -515,29 +534,36 @@ class StableDiffusion3Pipeline( # Broadcast timestep to match batch size timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype) - transformer_kwargs = { + positive_kwargs = { "hidden_states": latents, "timestep": timestep, "encoder_hidden_states": prompt_embeds, "pooled_projections": pooled_prompt_embeds, "return_dict": False, } - - noise_pred = self.transformer(**transformer_kwargs)[0] - - if do_cfg: - neg_transformer_kwargs = { + if do_true_cfg: + negative_kwargs = { "hidden_states": latents, "timestep": timestep, "encoder_hidden_states": negative_prompt_embeds, "pooled_projections": negative_pooled_prompt_embeds, "return_dict": False, } + else: + negative_kwargs = None + + # Predict noise with automatic CFG parallel handling + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg, + guidance_scale, + positive_kwargs, + negative_kwargs, + cfg_normalize, + ) + + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) - neg_noise_pred = self.transformer(**neg_transformer_kwargs)[0] - noise_pred = neg_noise_pred + self.guidance_scale * (noise_pred - neg_noise_pred) - # compute the previous noisy sample x_t -> x_t-1 - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] return latents def forward( @@ -644,14 +670,17 @@ class StableDiffusion3Pipeline( timesteps, num_inference_steps = self.prepare_timesteps(num_inference_steps, sigmas, latents.shape[1]) self._num_timesteps = len(timesteps) + # Denoising loop using diffuse method latents = self.diffuse( - prompt_embeds, - pooled_prompt_embeds, - negative_prompt_embeds, - negative_pooled_prompt_embeds, - latents, - timesteps, - do_cfg, + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds if do_cfg else None, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds if do_cfg else None, + do_true_cfg=do_cfg, + guidance_scale=self.guidance_scale, + cfg_normalize=False, ) self._current_timestep = None diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index 562d8eec5..b902bc692 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -7,7 +7,7 @@ import json import logging import os from collections.abc import Iterable -from typing import cast +from typing import Any, cast import PIL.Image import torch @@ -18,12 +18,14 @@ from transformers import AutoTokenizer, UMT5EncoderModel from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.inputs.data import OmniTextPrompt +from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) @@ -184,7 +186,7 @@ def get_wan22_pre_process_func( return pre_process_func -class Wan22Pipeline(nn.Module): +class Wan22Pipeline(nn.Module, CFGParallelMixin): def __init__( self, *, @@ -585,26 +587,44 @@ class Wan22Pipeline(nn.Module): latent_model_input = latents.to(dtype) timestep = t.expand(latents.shape[0]) - noise_pred = current_model( - hidden_states=latent_model_input, - timestep=timestep, - encoder_hidden_states=prompt_embeds, - attention_kwargs=attention_kwargs, - return_dict=False, - )[0] + do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None + # Prepare kwargs for positive and negative predictions + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": negative_prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + else: + negative_kwargs = None - if current_guidance_scale > 1.0 and negative_prompt_embeds is not None: - noise_uncond = current_model( - hidden_states=latent_model_input, - timestep=timestep, - encoder_hidden_states=negative_prompt_embeds, - attention_kwargs=attention_kwargs, - return_dict=False, - )[0] - noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond) + # Predict noise with automatic CFG parallel handling + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=current_guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + # Wan2.2 is prone to out of memory errors when predicting large videos + # so we empty the cache here to avoid OOM before vae decoding. + if current_omni_platform.is_available(): + current_omni_platform.empty_cache() self._current_timestep = None # For I2V mode: blend final latents with condition @@ -629,6 +649,21 @@ class Wan22Pipeline(nn.Module): return DiffusionOutput(output=output) + def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor: + """ + Forward pass through transformer to predict noise. + + Args: + current_model: The transformer model to use (transformer or transformer_2) + **kwargs: Arguments to pass to the transformer + + Returns: + Predicted noise tensor + """ + if current_model is None: + current_model = self.transformer + return current_model(**kwargs)[0] + def encode_prompt( self, prompt: str | list[str], diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index ea045e5d6..1aed9b75d 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging import os from collections.abc import Iterable -from typing import cast +from typing import Any, cast import numpy as np import PIL.Image @@ -18,6 +18,7 @@ from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportImageInput @@ -29,6 +30,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( ) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.inputs.data import OmniTextPrompt +from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) @@ -136,7 +138,7 @@ def get_wan22_i2v_pre_process_func( return pre_process_func -class Wan22I2VPipeline(nn.Module, SupportImageInput): +class Wan22I2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin): """ Wan2.2 Image-to-Video Pipeline. @@ -484,31 +486,46 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): latent_model_input = torch.cat([latents, condition], dim=1).to(dtype) timestep = t.expand(latents.shape[0]) - # Forward pass - noise_pred = current_model( - hidden_states=latent_model_input, - timestep=timestep, - encoder_hidden_states=prompt_embeds, - encoder_hidden_states_image=image_embeds, - attention_kwargs=attention_kwargs, - return_dict=False, - )[0] + do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None + # Prepare kwargs for positive and negative predictions + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": prompt_embeds, + "encoder_hidden_states_image": image_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": negative_prompt_embeds, + "encoder_hidden_states_image": image_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + else: + negative_kwargs = None - # Classifier-free guidance - if current_guidance_scale > 1.0 and negative_prompt_embeds is not None: - noise_uncond = current_model( - hidden_states=latent_model_input, - timestep=timestep, - encoder_hidden_states=negative_prompt_embeds, - encoder_hidden_states_image=image_embeds, - attention_kwargs=attention_kwargs, - return_dict=False, - )[0] - noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond) + # Predict noise with automatic CFG parallel handling + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=current_guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) - # Scheduler step - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + # Wan2.2 is prone to out of memory errors when predicting large videos + # so we empty the cache here to avoid OOM before vae decoding. + if current_omni_platform.is_available(): + current_omni_platform.empty_cache() self._current_timestep = None # For expand_timesteps mode, blend final latents with condition @@ -533,6 +550,21 @@ class Wan22I2VPipeline(nn.Module, SupportImageInput): return DiffusionOutput(output=output) + def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor: + """ + Forward pass through transformer to predict noise. + + Args: + current_model: The transformer model to use (transformer or transformer_2) + **kwargs: Arguments to pass to the transformer + + Returns: + Predicted noise tensor + """ + if current_model is None: + current_model = self.transformer + return current_model(**kwargs)[0] + def encode_prompt( self, prompt: str | list[str], diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py index 5944b678a..d32b7d697 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py @@ -19,7 +19,7 @@ from __future__ import annotations import logging import os from collections.abc import Iterable -from typing import cast +from typing import Any, cast import numpy as np import PIL.Image @@ -31,6 +31,7 @@ from transformers import AutoTokenizer, UMT5EncoderModel from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig +from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.models.interface import SupportImageInput @@ -42,6 +43,7 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( ) from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.inputs.data import OmniTextPrompt +from vllm_omni.platforms import current_omni_platform logger = logging.getLogger(__name__) @@ -126,7 +128,7 @@ def get_wan22_ti2v_pre_process_func( return pre_process_func -class Wan22TI2VPipeline(nn.Module, SupportImageInput): +class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin): """ Wan2.2 Text-Image-to-Video (TI2V) Pipeline. @@ -399,29 +401,44 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten() timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) - # Forward pass - noise_pred = self.transformer( - hidden_states=latent_model_input, - timestep=timestep, - encoder_hidden_states=prompt_embeds, - attention_kwargs=attention_kwargs, - return_dict=False, - )[0] + do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None + # Prepare kwargs for positive and negative predictions + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": self.transformer, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": negative_prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": self.transformer, + } + else: + negative_kwargs = None - # Classifier-free guidance - if guidance_scale > 1.0 and negative_prompt_embeds is not None: - noise_uncond = self.transformer( - hidden_states=latent_model_input, - timestep=timestep, - encoder_hidden_states=negative_prompt_embeds, - attention_kwargs=attention_kwargs, - return_dict=False, - )[0] - noise_pred = noise_uncond + guidance_scale * (noise_pred - noise_uncond) + # Predict noise with automatic CFG parallel handling + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) - # Scheduler step - latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + # Wan2.2 is prone to out of memory errors when predicting large videos + # so we empty the cache here to avoid OOM before vae decoding. + if current_omni_platform.is_available(): + current_omni_platform.empty_cache() self._current_timestep = None # For I2V mode, blend final latents with condition @@ -446,6 +463,21 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput): return DiffusionOutput(output=output) + def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor: + """ + Forward pass through transformer to predict noise. + + Args: + current_model: The transformer model to use + **kwargs: Arguments to pass to the transformer + + Returns: + Predicted noise tensor + """ + if current_model is None: + current_model = self.transformer + return current_model(**kwargs)[0] + def encode_prompt( self, prompt: str | list[str], diff --git a/vllm_omni/entrypoints/cli/serve.py b/vllm_omni/entrypoints/cli/serve.py index 5613bdaeb..083114ee4 100644 --- a/vllm_omni/entrypoints/cli/serve.py +++ b/vllm_omni/entrypoints/cli/serve.py @@ -225,7 +225,12 @@ class OmniServeCommand(CLISubcommand): help="Scheduler flow_shift for video models (e.g., 5.0 for 720p, 12.0 for 480p).", ) omni_config_group.add_argument( - "--cfg-parallel-size", type=int, default=1, help="Number of GPUs for CFG parallel computation" + "--cfg-parallel-size", + type=int, + default=1, + choices=[1, 2], + help="Number of devices for CFG parallel computation for diffusion models. " + "Equivalent to setting DiffusionParallelConfig.cfg_parallel_size.", ) return serve_parser From 425952e7d33c62cf38d0956eb457ad776c3be779 Mon Sep 17 00:00:00 2001 From: Markus / Mark <46672778+marksverdhei@users.noreply.github.com> Date: Sat, 31 Jan 2026 01:00:03 +0100 Subject: [PATCH 80/83] [BugFix] Fix Qwen3 TTS 0.6B profile run hang (#995) (#1082) --- vllm_omni/model_executor/models/qwen3_tts/qwen3_tts.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts.py b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts.py index 927ec0a90..8514a725d 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts.py +++ b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts.py @@ -125,6 +125,14 @@ class Qwen3TTSModelForGeneration(nn.Module): if isinstance(value, list) and len(value) > 0: runtime_additional_information[key] = value[0] + # During profile/warmup runs, text is empty and no real inputs exist. + # Cap generation steps so the full pipeline executes (preserving + # KV-cache profiling behaviour) but exits quickly even if the model + # cannot converge from degenerate dummy inputs. + if not text: + logger.info("Profile run detected (empty text). Capping max_new_tokens to 2.") + runtime_additional_information["max_new_tokens"] = 2 + # Call the appropriate generation method based on task_type if task_type == "CustomVoice": result = self.model.generate_custom_voice( From e503a4194128d12f7153306b297c03e46f86d698 Mon Sep 17 00:00:00 2001 From: TJian Date: Sat, 31 Jan 2026 09:46:23 +0800 Subject: [PATCH 81/83] [CI] [ROCm] Quick fix amd ci (#1116) Signed-off-by: tjtanaa --- .buildkite/test-amd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 930ef7a40..fac5c7268 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -113,4 +113,4 @@ steps: - export GPU_ARCHS=gfx942 - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/online_serving/test_i2i_multi_image_input.py + - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py From 07f4c3fdd6a628ae37a640029de872efcc2aa926 Mon Sep 17 00:00:00 2001 From: wangyu <53896905+yenuo26@users.noreply.github.com> Date: Sat, 31 Jan 2026 10:15:26 +0800 Subject: [PATCH 82/83] [Bugfix] fix benchmark audio timing error and add benchmark test (#1109) Signed-off-by: wangyu31577 Co-authored-by: wangyu31577 Co-authored-by: Hongsheng Liu --- .buildkite/pipeline.yml | 58 ++++++++++++++++++++++------ docs/cli/bench/serve.md | 2 +- tests/benchmarks/test_serve_cli.py | 60 +++++++++++++++++++++++++++++ vllm_omni/benchmarks/patch/patch.py | 4 +- 4 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 tests/benchmarks/test_serve_cli.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 25b14ce67..a2d0cdfe0 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -178,24 +178,60 @@ steps: volumes: - "/fsx/hf_cache:/fsx/hf_cache" + + - label: "Benchmark Test" + timeout_in_minutes: 15 + depends_on: image-build + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/benchmarks/test_serve_cli.py + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: "Omni Model Test" timeout_in_minutes: 15 depends_on: image-build commands: - - export VLLM_LOGGING_LEVEL=DEBUG - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py + - export VLLM_LOGGING_LEVEL=DEBUG + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py agents: queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU plugins: - - docker#v5.2.0: - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT - always-pull: true - propagate-environment: true - environment: - - "HF_HOME=/fsx/hf_cache" - volumes: - - "/fsx/hf_cache:/fsx/hf_cache" + - docker#v5.2.0: + image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + always-pull: true + propagate-environment: true + environment: + - "HF_HOME=/fsx/hf_cache" + volumes: + - "/fsx/hf_cache:/fsx/hf_cache" - label: "Omni Model Test with H100" timeout_in_minutes: 30 diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md index bcb1df74c..cc47bfc3c 100644 --- a/docs/cli/bench/serve.md +++ b/docs/cli/bench/serve.md @@ -128,7 +128,7 @@ Specify to save benchmark results to a json file Second item: videos with resolution 720x1280 and 16 frames Third item: audios with 1s duration and 5 channels w.p. 0.1 OBS.: If the probabilities do not sum to 1, they are normalized. - Only the random-mm mode supports this parameter. + Only the random-mm mode supports this parameter ## Usage Examples diff --git a/tests/benchmarks/test_serve_cli.py b/tests/benchmarks/test_serve_cli.py new file mode 100644 index 000000000..2c624b8e7 --- /dev/null +++ b/tests/benchmarks/test_serve_cli.py @@ -0,0 +1,60 @@ +import subprocess +from pathlib import Path + +import pytest + +from tests.conftest import OmniServer + +models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"] +stage_configs = [str(Path(__file__).parent.parent / "e2e" / "stage_configs" / "qwen3_omni_ci.yaml")] + +# Create parameter combinations for model and stage config +test_params = [(model, stage_config) for model in models for stage_config in stage_configs] + + +@pytest.fixture(scope="module") +def omni_server(request): + """Start vLLM-Omni server as a subprocess with actual model weights. + Uses session scope so the server starts only once for the entire test session. + Multi-stage initialization can take 10-20+ minutes. + """ + model, stage_config_path = request.param + + print(f"Starting OmniServer with model: {model}") + print("This may take 10-20+ minutes for initialization...") + + with OmniServer(model, ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"]) as server: + print("OmniServer started successfully") + yield server + print("OmniServer stopped") + + +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +def test_bench_serve_chat(omni_server): + command = [ + "vllm", + "bench", + "serve", + "--omni", + "--model", + omni_server.model, + "--port", + str(omni_server.port), + "--dataset-name", + "random", + "--random-input-len", + "32", + "--random-output-len", + "4", + "--num-prompts", + "5", + "--endpoint", + "/v1/chat/completions", + "--backend", + "openai-chat-omni", + ] + result = subprocess.run(command, capture_output=True, text=True) + print(result.stdout) + print(result.stderr) + + assert result.returncode == 0, f"Benchmark failed: {result.stderr}" diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index 6591af014..cf2e5c195 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -108,7 +108,7 @@ async def async_request_openai_chat_omni_completions( output.prompt_len = request_func_input.prompt_len generated_text = "" - generated_audio = "" + generated_audio = None ttft = 0.0 st = time.perf_counter() output.start_time = st @@ -160,7 +160,7 @@ async def async_request_openai_chat_omni_completions( if generated_audio is None: generated_audio = seg else: - generated_audio = seg + generated_audio + generated_audio = generated_audio + seg elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") From 2e6895ff0ed15eb5182ee0f191f75dbd2a90c27f Mon Sep 17 00:00:00 2001 From: Juan Pablo Zuluaga <46724788+JuanPZuluaga@users.noreply.github.com> Date: Sat, 31 Jan 2026 03:42:09 +0100 Subject: [PATCH 83/83] [Bugfix][Qwen3TTS] Load speaker_id/voices from model configuration (#1079) Signed-off-by: pablo Co-authored-by: Hongsheng Liu Co-authored-by: WeiQing Chen <40507679+david6666666@users.noreply.github.com> --- examples/online_serving/qwen3_tts/README.md | 3 ++ .../openai_api/test_serving_speech.py | 39 ++++++++++++++++- vllm_omni/entrypoints/openai/api_server.py | 19 ++++++++ .../entrypoints/openai/serving_speech.py | 43 +++++++++++++------ 4 files changed, 89 insertions(+), 15 deletions(-) diff --git a/examples/online_serving/qwen3_tts/README.md b/examples/online_serving/qwen3_tts/README.md index d4626e7c0..00f7ec2c9 100644 --- a/examples/online_serving/qwen3_tts/README.md +++ b/examples/online_serving/qwen3_tts/README.md @@ -75,6 +75,9 @@ curl -X POST http://localhost:8000/v1/audio/speech \ "voice": "Vivian", "instructions": "Speak with great enthusiasm" }' --output excited.wav + +# List available voices in CustomVoice models +curl http://localhost:8000/v1/audio/voices ``` ## API Reference diff --git a/tests/entrypoints/openai_api/test_serving_speech.py b/tests/entrypoints/openai_api/test_serving_speech.py index 675af4141..da73acb5a 100644 --- a/tests/entrypoints/openai_api/test_serving_speech.py +++ b/tests/entrypoints/openai_api/test_serving_speech.py @@ -201,6 +201,13 @@ def test_app(): app = FastAPI() app.add_api_route("/v1/audio/speech", speech_server.create_speech, methods=["POST"], response_model=None) + # Add list_voices endpoint + async def list_voices(): + speakers = sorted(speech_server.supported_speakers) if speech_server.supported_speakers else [] + return {"voices": speakers} + + app.add_api_route("/v1/audio/voices", list_voices, methods=["GET"]) + return app @@ -268,6 +275,11 @@ class TestSpeechAPI: assert isinstance(audio_obj, CreateAudio) assert audio_obj.speed == 2.5 + def test_list_voices_endpoint(self, client): + response = client.get("/v1/audio/voices") + assert response.status_code == 200 + assert "voices" in response.json() + class TestTTSMethods: """Unit tests for TTS validation and parameter building.""" @@ -311,9 +323,9 @@ class TestTTSMethods: req = OpenAICreateSpeechRequest(input="Hello", language="InvalidLang") assert "Invalid language" in speech_server._validate_tts_request(req) - # Invalid speaker + # When no speakers loaded, any voice is accepted (unconstrained) req = OpenAICreateSpeechRequest(input="Hello", voice="Invalid") - assert "Invalid speaker" in speech_server._validate_tts_request(req) + assert speech_server._validate_tts_request(req) is None # Valid request req = OpenAICreateSpeechRequest(input="Hello", voice="Vivian") @@ -342,3 +354,26 @@ class TestTTSMethods: assert params["speaker"] == ["Ryan"] assert params["language"] == ["English"] assert params["task_type"] == ["CustomVoice"] + + def test_load_supported_speakers(self): + """Test _load_supported_speakers.""" + mock_engine_client = MagicMock() + mock_engine_client.errored = False + mock_engine_client.stage_list = None + + # Mock talker_config with mixed-case speaker names + mock_talker_config = MagicMock() + mock_talker_config.spk_id = {"Ryan": 0, "Vivian": 1, "Aiden": 2} + mock_engine_client.model_config.hf_config.talker_config = mock_talker_config + + mock_models = MagicMock() + mock_models.is_base_model.return_value = True + + server = OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=MagicMock(), + ) + + # Verify speakers are normalized to lowercase + assert server.supported_speakers == {"ryan", "vivian", "aiden"} diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index 556ed8cfe..e428a6276 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -762,6 +762,25 @@ async def create_speech(request: OpenAICreateSpeechRequest, raw_request: Request raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e +@router.get( + "/v1/audio/voices", + responses={ + HTTPStatus.OK.value: {"model": dict}, + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def list_voices(raw_request: Request): + """List available TTS voices/speakers from the loaded model.""" + handler = Omnispeech(raw_request) + if handler is None: + return base(raw_request).create_error_response(message="The model does not support Speech API") + + speakers = sorted(handler.supported_speakers) if handler.supported_speakers else [] + return JSONResponse(content={"voices": speakers}) + + # Health and Model endpoints for diffusion mode diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 223557aab..cb9f6f8a8 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -19,17 +19,6 @@ logger = init_logger(__name__) # TTS Configuration (currently supports Qwen3-TTS) _TTS_MODEL_STAGES: set[str] = {"qwen3_tts"} -_TTS_SPEAKERS: set[str] = { - "Vivian", - "Serena", - "Uncle_Fu", - "Dylan", - "Eric", - "Ryan", - "Aiden", - "Ono_Anna", - "Sohee", -} _TTS_LANGUAGES: set[str] = { "Auto", "Chinese", @@ -49,6 +38,30 @@ _TTS_MAX_NEW_TOKENS_MAX = 4096 class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Load supported speakers + self.supported_speakers = self._load_supported_speakers() + logger.info(f"Loaded {len(self.supported_speakers)} supported speakers: {sorted(self.supported_speakers)}") + + def _load_supported_speakers(self) -> set[str]: + """Load supported speakers (case-insensitive) from the model configuration.""" + try: + talker_config = self.engine_client.model_config.hf_config.talker_config + + # Check for speakers in either spk_id or speaker_id + for attr_name in ["spk_id", "speaker_id"]: + speakers_dict = getattr(talker_config, attr_name, None) + if speakers_dict and isinstance(speakers_dict, dict): + # Normalize to lowercase for case-insensitive matching + return {speaker.lower() for speaker in speakers_dict.keys()} + + logger.warning("No speakers found in talker_config (checked spk_id and speaker_id)") + except Exception as e: + logger.warning(f"Could not load speakers from model config: {e}") + + return set() + def _is_tts_model(self) -> bool: """Check if the current model is a supported TTS model.""" stage_list = getattr(self.engine_client, "stage_list", None) @@ -63,6 +76,10 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): """Validate TTS request parameters. Returns error message or None.""" task_type = request.task_type or "CustomVoice" + # Normalize voice to lowercase for case-insensitive matching + if request.voice is not None: + request.voice = request.voice.lower() + # Validate input is not empty if not request.input or not request.input.strip(): return "Input text cannot be empty" @@ -73,8 +90,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): # Validate speaker for CustomVoice task if task_type == "CustomVoice" and request.voice is not None: - if request.voice not in _TTS_SPEAKERS: - return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(_TTS_SPEAKERS))}" + if self.supported_speakers and request.voice not in self.supported_speakers: + return f"Invalid speaker '{request.voice}'. Supported: {', '.join(sorted(self.supported_speakers))}" # Validate Base task requirements if task_type == "Base":